{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:34:04.812230400Z",
     "start_time": "2023-12-09T01:34:01.763236800Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "data": {
      "text/plain": "   customerID  gender  SeniorCitizen Partner Dependents  tenure PhoneService  \\\n0  7590-VHVEG  Female              0     Yes         No       1           No   \n1  5575-GNVDE    Male              0      No         No      34          Yes   \n2  3668-QPYBK    Male              0      No         No       2          Yes   \n3  7795-CFOCW    Male              0      No         No      45           No   \n4  9237-HQITU  Female              0      No         No       2          Yes   \n\n      MultipleLines InternetService OnlineSecurity  ... DeviceProtection  \\\n0  No phone service             DSL             No  ...               No   \n1                No             DSL            Yes  ...              Yes   \n2                No             DSL            Yes  ...               No   \n3  No phone service             DSL            Yes  ...              Yes   \n4                No     Fiber optic             No  ...               No   \n\n  TechSupport StreamingTV StreamingMovies        Contract PaperlessBilling  \\\n0          No          No              No  Month-to-month              Yes   \n1          No          No              No        One year               No   \n2          No          No              No  Month-to-month              Yes   \n3         Yes          No              No        One year               No   \n4          No          No              No  Month-to-month              Yes   \n\n               PaymentMethod MonthlyCharges  TotalCharges Churn  \n0           Electronic check          29.85         29.85    No  \n1               Mailed check          56.95        1889.5    No  \n2               Mailed check          53.85        108.15   Yes  \n3  Bank transfer (automatic)          42.30       1840.75    No  \n4           Electronic check          70.70        151.65   Yes  \n\n[5 rows x 21 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>customerID</th>\n      <th>gender</th>\n      <th>SeniorCitizen</th>\n      <th>Partner</th>\n      <th>Dependents</th>\n      <th>tenure</th>\n      <th>PhoneService</th>\n      <th>MultipleLines</th>\n      <th>InternetService</th>\n      <th>OnlineSecurity</th>\n      <th>...</th>\n      <th>DeviceProtection</th>\n      <th>TechSupport</th>\n      <th>StreamingTV</th>\n      <th>StreamingMovies</th>\n      <th>Contract</th>\n      <th>PaperlessBilling</th>\n      <th>PaymentMethod</th>\n      <th>MonthlyCharges</th>\n      <th>TotalCharges</th>\n      <th>Churn</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>7590-VHVEG</td>\n      <td>Female</td>\n      <td>0</td>\n      <td>Yes</td>\n      <td>No</td>\n      <td>1</td>\n      <td>No</td>\n      <td>No phone service</td>\n      <td>DSL</td>\n      <td>No</td>\n      <td>...</td>\n      <td>No</td>\n      <td>No</td>\n      <td>No</td>\n      <td>No</td>\n      <td>Month-to-month</td>\n      <td>Yes</td>\n      <td>Electronic check</td>\n      <td>29.85</td>\n      <td>29.85</td>\n      <td>No</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>5575-GNVDE</td>\n      <td>Male</td>\n      <td>0</td>\n      <td>No</td>\n      <td>No</td>\n      <td>34</td>\n      <td>Yes</td>\n      <td>No</td>\n      <td>DSL</td>\n      <td>Yes</td>\n      <td>...</td>\n      <td>Yes</td>\n      <td>No</td>\n      <td>No</td>\n      <td>No</td>\n      <td>One year</td>\n      <td>No</td>\n      <td>Mailed check</td>\n      <td>56.95</td>\n      <td>1889.5</td>\n      <td>No</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3668-QPYBK</td>\n      <td>Male</td>\n      <td>0</td>\n      <td>No</td>\n      <td>No</td>\n      <td>2</td>\n      <td>Yes</td>\n      <td>No</td>\n      <td>DSL</td>\n      <td>Yes</td>\n      <td>...</td>\n      <td>No</td>\n      <td>No</td>\n      <td>No</td>\n      <td>No</td>\n      <td>Month-to-month</td>\n      <td>Yes</td>\n      <td>Mailed check</td>\n      <td>53.85</td>\n      <td>108.15</td>\n      <td>Yes</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>7795-CFOCW</td>\n      <td>Male</td>\n      <td>0</td>\n      <td>No</td>\n      <td>No</td>\n      <td>45</td>\n      <td>No</td>\n      <td>No phone service</td>\n      <td>DSL</td>\n      <td>Yes</td>\n      <td>...</td>\n      <td>Yes</td>\n      <td>Yes</td>\n      <td>No</td>\n      <td>No</td>\n      <td>One year</td>\n      <td>No</td>\n      <td>Bank transfer (automatic)</td>\n      <td>42.30</td>\n      <td>1840.75</td>\n      <td>No</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>9237-HQITU</td>\n      <td>Female</td>\n      <td>0</td>\n      <td>No</td>\n      <td>No</td>\n      <td>2</td>\n      <td>Yes</td>\n      <td>No</td>\n      <td>Fiber optic</td>\n      <td>No</td>\n      <td>...</td>\n      <td>No</td>\n      <td>No</td>\n      <td>No</td>\n      <td>No</td>\n      <td>Month-to-month</td>\n      <td>Yes</td>\n      <td>Electronic check</td>\n      <td>70.70</td>\n      <td>151.65</td>\n      <td>Yes</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 21 columns</p>\n</div>"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"F:/机器学习数据集/电信客户流失数据/WA_Fn-UseC_-Telco-Customer-Churn.csv\")\n",
    "df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:34:38.822227600Z",
     "start_time": "2023-12-09T01:34:37.624147600Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 数据预处理"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "df.columns = df.columns.str.lower().str.replace(' ','_')\n",
    "str_columns = list(df.dtypes[df.dtypes=='object'].index)\n",
    "for col in str_columns:\n",
    "    df[col] = df[col].str.lower().str.replace(\" \",\"_\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:38:21.823744900Z",
     "start_time": "2023-12-09T01:38:21.683744100Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "data": {
      "text/plain": "   customerid  gender  seniorcitizen partner dependents  tenure phoneservice  \\\n0  7590-vhveg  female              0     yes         no       1           no   \n1  5575-gnvde    male              0      no         no      34          yes   \n2  3668-qpybk    male              0      no         no       2          yes   \n3  7795-cfocw    male              0      no         no      45           no   \n4  9237-hqitu  female              0      no         no       2          yes   \n\n      multiplelines internetservice onlinesecurity  ... deviceprotection  \\\n0  no_phone_service             dsl             no  ...               no   \n1                no             dsl            yes  ...              yes   \n2                no             dsl            yes  ...               no   \n3  no_phone_service             dsl            yes  ...              yes   \n4                no     fiber_optic             no  ...               no   \n\n  techsupport streamingtv streamingmovies        contract paperlessbilling  \\\n0          no          no              no  month-to-month              yes   \n1          no          no              no        one_year               no   \n2          no          no              no  month-to-month              yes   \n3         yes          no              no        one_year               no   \n4          no          no              no  month-to-month              yes   \n\n               paymentmethod monthlycharges  totalcharges churn  \n0           electronic_check          29.85         29.85    no  \n1               mailed_check          56.95        1889.5    no  \n2               mailed_check          53.85        108.15   yes  \n3  bank_transfer_(automatic)          42.30       1840.75    no  \n4           electronic_check          70.70        151.65   yes  \n\n[5 rows x 21 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>customerid</th>\n      <th>gender</th>\n      <th>seniorcitizen</th>\n      <th>partner</th>\n      <th>dependents</th>\n      <th>tenure</th>\n      <th>phoneservice</th>\n      <th>multiplelines</th>\n      <th>internetservice</th>\n      <th>onlinesecurity</th>\n      <th>...</th>\n      <th>deviceprotection</th>\n      <th>techsupport</th>\n      <th>streamingtv</th>\n      <th>streamingmovies</th>\n      <th>contract</th>\n      <th>paperlessbilling</th>\n      <th>paymentmethod</th>\n      <th>monthlycharges</th>\n      <th>totalcharges</th>\n      <th>churn</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>7590-vhveg</td>\n      <td>female</td>\n      <td>0</td>\n      <td>yes</td>\n      <td>no</td>\n      <td>1</td>\n      <td>no</td>\n      <td>no_phone_service</td>\n      <td>dsl</td>\n      <td>no</td>\n      <td>...</td>\n      <td>no</td>\n      <td>no</td>\n      <td>no</td>\n      <td>no</td>\n      <td>month-to-month</td>\n      <td>yes</td>\n      <td>electronic_check</td>\n      <td>29.85</td>\n      <td>29.85</td>\n      <td>no</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>5575-gnvde</td>\n      <td>male</td>\n      <td>0</td>\n      <td>no</td>\n      <td>no</td>\n      <td>34</td>\n      <td>yes</td>\n      <td>no</td>\n      <td>dsl</td>\n      <td>yes</td>\n      <td>...</td>\n      <td>yes</td>\n      <td>no</td>\n      <td>no</td>\n      <td>no</td>\n      <td>one_year</td>\n      <td>no</td>\n      <td>mailed_check</td>\n      <td>56.95</td>\n      <td>1889.5</td>\n      <td>no</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3668-qpybk</td>\n      <td>male</td>\n      <td>0</td>\n      <td>no</td>\n      <td>no</td>\n      <td>2</td>\n      <td>yes</td>\n      <td>no</td>\n      <td>dsl</td>\n      <td>yes</td>\n      <td>...</td>\n      <td>no</td>\n      <td>no</td>\n      <td>no</td>\n      <td>no</td>\n      <td>month-to-month</td>\n      <td>yes</td>\n      <td>mailed_check</td>\n      <td>53.85</td>\n      <td>108.15</td>\n      <td>yes</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>7795-cfocw</td>\n      <td>male</td>\n      <td>0</td>\n      <td>no</td>\n      <td>no</td>\n      <td>45</td>\n      <td>no</td>\n      <td>no_phone_service</td>\n      <td>dsl</td>\n      <td>yes</td>\n      <td>...</td>\n      <td>yes</td>\n      <td>yes</td>\n      <td>no</td>\n      <td>no</td>\n      <td>one_year</td>\n      <td>no</td>\n      <td>bank_transfer_(automatic)</td>\n      <td>42.30</td>\n      <td>1840.75</td>\n      <td>no</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>9237-hqitu</td>\n      <td>female</td>\n      <td>0</td>\n      <td>no</td>\n      <td>no</td>\n      <td>2</td>\n      <td>yes</td>\n      <td>no</td>\n      <td>fiber_optic</td>\n      <td>no</td>\n      <td>...</td>\n      <td>no</td>\n      <td>no</td>\n      <td>no</td>\n      <td>no</td>\n      <td>month-to-month</td>\n      <td>yes</td>\n      <td>electronic_check</td>\n      <td>70.70</td>\n      <td>151.65</td>\n      <td>yes</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 21 columns</p>\n</div>"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:38:46.548624700Z",
     "start_time": "2023-12-09T01:38:46.477624400Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "data": {
      "text/plain": "   customerid  gender  seniorcitizen partner dependents  tenure phoneservice  \\\n0  7590-vhveg  female              0     yes         no       1           no   \n1  5575-gnvde    male              0      no         no      34          yes   \n2  3668-qpybk    male              0      no         no       2          yes   \n3  7795-cfocw    male              0      no         no      45           no   \n4  9237-hqitu  female              0      no         no       2          yes   \n\n      multiplelines internetservice onlinesecurity  ... deviceprotection  \\\n0  no_phone_service             dsl             no  ...               no   \n1                no             dsl            yes  ...              yes   \n2                no             dsl            yes  ...               no   \n3  no_phone_service             dsl            yes  ...              yes   \n4                no     fiber_optic             no  ...               no   \n\n  techsupport streamingtv streamingmovies        contract paperlessbilling  \\\n0          no          no              no  month-to-month              yes   \n1          no          no              no        one_year               no   \n2          no          no              no  month-to-month              yes   \n3         yes          no              no        one_year               no   \n4          no          no              no  month-to-month              yes   \n\n               paymentmethod monthlycharges  totalcharges churn  \n0           electronic_check          29.85         29.85     0  \n1               mailed_check          56.95        1889.5     0  \n2               mailed_check          53.85        108.15     1  \n3  bank_transfer_(automatic)          42.30       1840.75     0  \n4           electronic_check          70.70        151.65     1  \n\n[5 rows x 21 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>customerid</th>\n      <th>gender</th>\n      <th>seniorcitizen</th>\n      <th>partner</th>\n      <th>dependents</th>\n      <th>tenure</th>\n      <th>phoneservice</th>\n      <th>multiplelines</th>\n      <th>internetservice</th>\n      <th>onlinesecurity</th>\n      <th>...</th>\n      <th>deviceprotection</th>\n      <th>techsupport</th>\n      <th>streamingtv</th>\n      <th>streamingmovies</th>\n      <th>contract</th>\n      <th>paperlessbilling</th>\n      <th>paymentmethod</th>\n      <th>monthlycharges</th>\n      <th>totalcharges</th>\n      <th>churn</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>7590-vhveg</td>\n      <td>female</td>\n      <td>0</td>\n      <td>yes</td>\n      <td>no</td>\n      <td>1</td>\n      <td>no</td>\n      <td>no_phone_service</td>\n      <td>dsl</td>\n      <td>no</td>\n      <td>...</td>\n      <td>no</td>\n      <td>no</td>\n      <td>no</td>\n      <td>no</td>\n      <td>month-to-month</td>\n      <td>yes</td>\n      <td>electronic_check</td>\n      <td>29.85</td>\n      <td>29.85</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>5575-gnvde</td>\n      <td>male</td>\n      <td>0</td>\n      <td>no</td>\n      <td>no</td>\n      <td>34</td>\n      <td>yes</td>\n      <td>no</td>\n      <td>dsl</td>\n      <td>yes</td>\n      <td>...</td>\n      <td>yes</td>\n      <td>no</td>\n      <td>no</td>\n      <td>no</td>\n      <td>one_year</td>\n      <td>no</td>\n      <td>mailed_check</td>\n      <td>56.95</td>\n      <td>1889.5</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3668-qpybk</td>\n      <td>male</td>\n      <td>0</td>\n      <td>no</td>\n      <td>no</td>\n      <td>2</td>\n      <td>yes</td>\n      <td>no</td>\n      <td>dsl</td>\n      <td>yes</td>\n      <td>...</td>\n      <td>no</td>\n      <td>no</td>\n      <td>no</td>\n      <td>no</td>\n      <td>month-to-month</td>\n      <td>yes</td>\n      <td>mailed_check</td>\n      <td>53.85</td>\n      <td>108.15</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>7795-cfocw</td>\n      <td>male</td>\n      <td>0</td>\n      <td>no</td>\n      <td>no</td>\n      <td>45</td>\n      <td>no</td>\n      <td>no_phone_service</td>\n      <td>dsl</td>\n      <td>yes</td>\n      <td>...</td>\n      <td>yes</td>\n      <td>yes</td>\n      <td>no</td>\n      <td>no</td>\n      <td>one_year</td>\n      <td>no</td>\n      <td>bank_transfer_(automatic)</td>\n      <td>42.30</td>\n      <td>1840.75</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>9237-hqitu</td>\n      <td>female</td>\n      <td>0</td>\n      <td>no</td>\n      <td>no</td>\n      <td>2</td>\n      <td>yes</td>\n      <td>no</td>\n      <td>fiber_optic</td>\n      <td>no</td>\n      <td>...</td>\n      <td>no</td>\n      <td>no</td>\n      <td>no</td>\n      <td>no</td>\n      <td>month-to-month</td>\n      <td>yes</td>\n      <td>electronic_check</td>\n      <td>70.70</td>\n      <td>151.65</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 21 columns</p>\n</div>"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.churn = (df.churn=='yes').astype(int)\n",
    "df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:40:26.408925Z",
     "start_time": "2023-12-09T01:40:26.315925100Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": "customerid           object\ngender               object\nseniorcitizen         int64\npartner              object\ndependents           object\ntenure                int64\nphoneservice         object\nmultiplelines        object\ninternetservice      object\nonlinesecurity       object\nonlinebackup         object\ndeviceprotection     object\ntechsupport          object\nstreamingtv          object\nstreamingmovies      object\ncontract             object\npaperlessbilling     object\npaymentmethod        object\nmonthlycharges      float64\ntotalcharges         object\nchurn                 int32\ndtype: object"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:40:31.656613100Z",
     "start_time": "2023-12-09T01:40:31.633618Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "data": {
      "text/plain": "dtype('float64')"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.totalcharges = pd.to_numeric(df.totalcharges,errors='coerce')\n",
    "df.totalcharges = df.totalcharges.fillna(0)\n",
    "df.totalcharges.dtypes"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:40:52.601734500Z",
     "start_time": "2023-12-09T01:40:52.502729400Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "customerid          0\ngender              0\nseniorcitizen       0\npartner             0\ndependents          0\ntenure              0\nphoneservice        0\nmultiplelines       0\ninternetservice     0\nonlinesecurity      0\nonlinebackup        0\ndeviceprotection    0\ntechsupport         0\nstreamingtv         0\nstreamingmovies     0\ncontract            0\npaperlessbilling    0\npaymentmethod       0\nmonthlycharges      0\ntotalcharges        0\nchurn               0\ndtype: int64"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:40:53.342730500Z",
     "start_time": "2023-12-09T01:40:53.216733800Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 计算互信息"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "categorical = ['gender','seniorcitizen','partner','dependents','phoneservice','multiplelines'\n",
    "               ,'internetservice','onlinesecurity','onlinebackup','deviceprotection','techsupport',\n",
    "               'streamingtv','streamingmovies','contract','paperlessbilling','paymentmethod']\n",
    "numerical = ['tenure','monthlycharges','totalcharges']"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:40:55.987728900Z",
     "start_time": "2023-12-09T01:40:55.979733900Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "data": {
      "text/plain": "                        MI\ncontract          0.098453\nonlinesecurity    0.064677\ntechsupport       0.063021\ninternetservice   0.055574\nonlinebackup      0.046792\npaymentmethod     0.044519\ndeviceprotection  0.043917\nstreamingmovies   0.032001\nstreamingtv       0.031908\npaperlessbilling  0.019194\ndependents        0.014467\npartner           0.011454\nseniorcitizen     0.010577\nmultiplelines     0.000801\nphoneservice      0.000072\ngender            0.000037",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>MI</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>contract</th>\n      <td>0.098453</td>\n    </tr>\n    <tr>\n      <th>onlinesecurity</th>\n      <td>0.064677</td>\n    </tr>\n    <tr>\n      <th>techsupport</th>\n      <td>0.063021</td>\n    </tr>\n    <tr>\n      <th>internetservice</th>\n      <td>0.055574</td>\n    </tr>\n    <tr>\n      <th>onlinebackup</th>\n      <td>0.046792</td>\n    </tr>\n    <tr>\n      <th>paymentmethod</th>\n      <td>0.044519</td>\n    </tr>\n    <tr>\n      <th>deviceprotection</th>\n      <td>0.043917</td>\n    </tr>\n    <tr>\n      <th>streamingmovies</th>\n      <td>0.032001</td>\n    </tr>\n    <tr>\n      <th>streamingtv</th>\n      <td>0.031908</td>\n    </tr>\n    <tr>\n      <th>paperlessbilling</th>\n      <td>0.019194</td>\n    </tr>\n    <tr>\n      <th>dependents</th>\n      <td>0.014467</td>\n    </tr>\n    <tr>\n      <th>partner</th>\n      <td>0.011454</td>\n    </tr>\n    <tr>\n      <th>seniorcitizen</th>\n      <td>0.010577</td>\n    </tr>\n    <tr>\n      <th>multiplelines</th>\n      <td>0.000801</td>\n    </tr>\n    <tr>\n      <th>phoneservice</th>\n      <td>0.000072</td>\n    </tr>\n    <tr>\n      <th>gender</th>\n      <td>0.000037</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import mutual_info_score\n",
    "def calculate_mi(series):\n",
    "    return mutual_info_score(series,df.churn)\n",
    "df_mi=df[categorical].apply(calculate_mi)\n",
    "df_mi = df_mi.sort_values(ascending=False).to_frame(name='MI')\n",
    "df_mi"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:41:07.693108Z",
     "start_time": "2023-12-09T01:41:00.479447700Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 计算相关系数"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "data": {
      "text/plain": "tenure           -0.352229\nmonthlycharges    0.193356\ntotalcharges     -0.198324\ndtype: float64"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[numerical].corrwith(df.churn)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:41:27.592281500Z",
     "start_time": "2023-12-09T01:41:25.806815300Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 独热编码"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 0)\t1.0\n",
      "  (0, 3)\t1.0\n",
      "  (0, 5)\t1.0\n",
      "  (0, 8)\t1.0\n",
      "  (0, 10)\t1.0\n",
      "  (0, 13)\t29.85\n",
      "  (0, 15)\t1.0\n",
      "  (0, 19)\t1.0\n",
      "  (0, 20)\t1.0\n",
      "  (0, 24)\t1.0\n",
      "  (0, 26)\t1.0\n",
      "  (0, 29)\t1.0\n",
      "  (0, 31)\t1.0\n",
      "  (0, 33)\t0.0\n",
      "  (0, 34)\t1.0\n",
      "  (0, 37)\t1.0\n",
      "  (0, 40)\t1.0\n",
      "  (0, 43)\t1.0\n",
      "  (0, 44)\t29.85\n",
      "0       0\n",
      "1       0\n",
      "2       1\n",
      "3       0\n",
      "4       1\n",
      "       ..\n",
      "7038    0\n",
      "7039    0\n",
      "7040    0\n",
      "7041    1\n",
      "7042    0\n",
      "Name: churn, Length: 7043, dtype: int32\n"
     ]
    },
    {
     "data": {
      "text/plain": "['contract=month-to-month',\n 'contract=one_year',\n 'contract=two_year',\n 'dependents=no',\n 'dependents=yes',\n 'deviceprotection=no',\n 'deviceprotection=no_internet_service',\n 'deviceprotection=yes',\n 'gender=female',\n 'gender=male',\n 'internetservice=dsl',\n 'internetservice=fiber_optic',\n 'internetservice=no',\n 'monthlycharges',\n 'multiplelines=no',\n 'multiplelines=no_phone_service',\n 'multiplelines=yes',\n 'onlinebackup=no',\n 'onlinebackup=no_internet_service',\n 'onlinebackup=yes',\n 'onlinesecurity=no',\n 'onlinesecurity=no_internet_service',\n 'onlinesecurity=yes',\n 'paperlessbilling=no',\n 'paperlessbilling=yes',\n 'partner=no',\n 'partner=yes',\n 'paymentmethod=bank_transfer_(automatic)',\n 'paymentmethod=credit_card_(automatic)',\n 'paymentmethod=electronic_check',\n 'paymentmethod=mailed_check',\n 'phoneservice=no',\n 'phoneservice=yes',\n 'seniorcitizen',\n 'streamingmovies=no',\n 'streamingmovies=no_internet_service',\n 'streamingmovies=yes',\n 'streamingtv=no',\n 'streamingtv=no_internet_service',\n 'streamingtv=yes',\n 'techsupport=no',\n 'techsupport=no_internet_service',\n 'techsupport=yes',\n 'tenure',\n 'totalcharges']"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction import DictVectorizer\n",
    "df_dict = df[categorical+numerical].to_dict(orient='records') # 先将数据框中的数据转化为字典列表\n",
    "dv = DictVectorizer() # 字典向量化类\n",
    "dv.fit(df_dict)\n",
    "X = dv.transform(df_dict)\n",
    "y = df.churn\n",
    "print(X[0])\n",
    "print(y)\n",
    "dv.get_feature_names()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:42:16.095483500Z",
     "start_time": "2023-12-09T01:42:15.197484200Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 划分训练集和验证集"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "测试集精度:81.204%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=1)\n",
    "lr = LogisticRegression(random_state=1)\n",
    "lr.fit(X_train,y_train)\n",
    "print('测试集精度:{:.3f}%'.format(lr.score(X_test,y_test)*100))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:43:23.009664400Z",
     "start_time": "2023-12-09T01:43:20.845609400Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "data": {
      "text/plain": "81.204"
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred=lr.predict_proba(X_test)[:,1] # 从模型中做出预测\n",
    "churn = y_pred>=0.5 # 做出硬预测\n",
    "score=(churn==y_test).mean()*100 # 计算准确度\n",
    "round(score,3)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:43:25.295026Z",
     "start_time": "2023-12-09T01:43:25.146024900Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 4.1. 评估指标\n",
    "### 4.1.1 分类准确度"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "阈值:0.00, 准确度:24.645%\n",
      "阈值:0.10, 准确度:61.272%\n",
      "阈值:0.20, 准确度:71.721%\n",
      "阈值:0.30, 准确度:77.286%\n",
      "阈值:0.40, 准确度:79.387%\n",
      "阈值:0.50, 准确度:81.204%\n",
      "阈值:0.60, 准确度:81.317%\n",
      "阈值:0.70, 准确度:79.671%\n",
      "阈值:0.80, 准确度:76.547%\n",
      "阈值:0.90, 准确度:75.355%\n",
      "阈值:1.00, 准确度:75.355%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "thresholds = np.linspace(0,1,11) # 创建一个具有不同阈值的数组[0.0,0.1,0.2...]\n",
    "for t in thresholds:\n",
    "    churn = y_pred >t\n",
    "    acc = accuracy_score(y_test,churn)\n",
    "    print(\"阈值:{:.2f}, 准确度:{:.3f}%\".format(t,acc*100))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:43:28.801123200Z",
     "start_time": "2023-12-09T01:43:28.756942900Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "data": {
      "text/plain": "[<matplotlib.lines.Line2D at 0x26714c00130>]"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAfAUlEQVR4nO3deXxV533n8c9PVwsSSAItICQWYcwOBhuB7WzGqR0vSUycOFM7mewZ1804r07amdrpkjbTmU7cTF9J2rhlHJdmaWumSWyWCYnjpLXjLA6LEWYTWAYD2kBi0YbWe3/zx73GsiLMBa50dM/9vl+v+9I99zw++j0Ivnr8nPOcY+6OiIikv6ygCxARkdRQoIuIhIQCXUQkJBToIiIhoUAXEQmJ7KC+cVlZmVdXVwf17UVE0tLOnTvb3L18pH2BBXp1dTU7duwI6tuLiKQlMzt6oX2achERCQkFuohISCjQRURCQoEuIhISCnQRkZBQoIuIhIQCXUQkJAK7Dl1kvHN3+gZj9A5E6RmI0tMf/9o7EKWnPxb/bCBKb3+U3sHX98ccsrOMyGsvM7KyjOys+NeIDXmfBZGsLCIWf1+Qm03ZpDzKC/MomZhLJMuC/mOQNKJAl4w3EI2xr6mDHa+eZvurp9l9vJ2O3gF6BqIE+biALIOSibnnA75sUh5lk3KHvM9T+MsbKNAl43T2DrDr2NlEgJ9h1/Ez9A7EAJhdWsCNc0spnZhLfm6ECTkR8nMi5OfGv04Y8j7+edYb2kzIjmAGMYfBWIxY7PWvUfc3vI9GPf41Fn8NxmJ090Vp6+qLvzr7aO3qo7Wzn7auPg63dtPW1UffYOw3+pRlUDk5n7fPK2PNgqm89eoyJuXpn3emSeonbma3A18DIsDj7v6lYfuLgX8CZiWO+b/d/R9TXKvIZWlp72XH0dPsePUM2189zYHmDmIeD8EllcXct3oWq6pLqJk9halFE1LyPSMGkazIa1spOSbEp4G6+gZp6+qntbPvfPi3dvZx6EQnW3Y388S24+REjJrZJaxZUM6aBVOZP20SZhrBh51d7BF0ZhYBDgG3Ag3AduA+d98/pM0fAcXu/pCZlQMHgQp377/QcWtqalz3cpFU6x2IcrClk71N7ecDvOFMDwD5ORGumz2ZmtklrKouYcWsyaEbxfYPxth59AzPHjrJcwdbqWvpBKCyeAI3LZjKmgXlGr2nOTPb6e41I+1L5qe6Gqh398OJg20A1gL7h7RxoNDiQ4BJwGlg8IqqFnkT7k5zey8Hmjvir5ZODjR38GpbN7HEGKVsUh6rqqfwibfOYVX1FBZNLyInEu4Lu3Kzs7hxbik3zi3l83csorm9h2cPtvLswZNs2d3EE9uOkRMxVlW/PnqfN1Wj97BIZoR+D3C7u386sf0R4Hp3f3BIm0JgM7AQKAR+291/MMKx7gfuB5g1a9bKo0cveNMwkfN6B6IcOtGZCO/417qWTtp7Bs63mVmSz6KKIhZOL2Lx9EIWTy9mZkm+gmqIoaP3Z+taOXgiPnqvmpzPTQvKec+y6Vx/ValOro5zbzZCTybQPwjcNizQV7v7Z4e0uQd4K/D7wFzgGWC5u3dc6LiacpELOXqqm2cPtrLt1dPUNXdwZMiouyA3woKKQhZNL4q/KgpZUFFI4YScYItOQ01ne3juUHz0/vzLbZzrjzK1MI/3Lq9k7YpKllUV6xfiOHSlUy4NwMwh2zOApmFtPgF8yeO/HerN7Ajx0fq2y6hXMkzvQJQXDp/i2YOtPHeolSNt3UB85Li4soh3X1PJ4umFLKwoYlZJAVkaQaZE5eR87ls9i/tWz6KnP8pP606wubaJ7/zqKP/w8yPMKZvIXcsruWtFJXPLJwVdriQhmRF6NvGTor8FNBI/Kfohd983pM3fAyfc/c/NbBrwIvERetuFjqsRemZ7bRT+7MGT/OrwKXoHYuQl5n9vTpy8m106MegyM1L7uQF+tK+ZTbVN/OrwKdxhWVUxdy2v5L3LK6koTs2VQHJ5rmjKJXGAO4GvEr/+ar27/08zewDA3deZWSXwTWA6YMRH6//0ZsdUoGeW3oEovz5ymmcPnuTZg6+PwqtLC1iTCPAbriplQk7qLvGTK3eio5ctu5vYsruJ3Q3tmMH1c0pYu6KKO5ZWMLkgN+gSM84VB/poUKCHWzTmHGnr4pevnOLf6944Cr/hqlJuTlxhUV2mUXi6ONLWzebaJjbtbuRwazc5EeOm+VO5a0Ulty6aRn6ufhmPBQW6jKqBaIyXT3Sxr6mdfU0d7G1sZ39zB+f6o0B89eXNC6Zy04JybtQoPO25O/uaOthU28iW3c20dPQyKS+b25dW8P5rq7jhqlKd5xhFCnRJmaELd/Y2drCvqZ26lk76E8vRC3IjLKksYkllMUsqi6ipLmGORuGhFYs5vz5ymqd2NbB1TwtdfYNUFk9g7bVVvP/aKuZNKwy6xNBRoMtlcXdeamhn17Ez7E2MvF8+2UU0cQ1h0YRsllYVs7QqHt5Lq4qpLp2o65gzVE9/lGcOnOCpFxv42cttRGPO0qoi7r52Bnctr6S8MC/oEkNBgS6XpLm9hydfbOR7OxvOn7wsm5T7enBXxkN8xhQt3JGRtXb2sWV3E0/tamRPYzuRLOMd88q4+7oZvGvxNE27XQEFulxU70CUH+8/wfd2NvDzl1uJOayeU8I9K2dw0/xyphbmKbzlsrx8opMndzWyaVcjTe3x+fY7l1Vw97UzuH5OiebbL5ECXUbk7uxuaOe7O46zZXcTHb2DVE3O5wPXVfGBlTN0HbikVCzmvHDkFE+92MjWPc1090epmpzPrYunUVM9hVXVJUxL0d0uw0yBLm9wsqOXp3bFp1RePtnFhJws7lg6nXtWzuBGXaEgY6CnP8qP97ewcVcjLxw+Tc9A/IqomSX5rJpdQk11CauqpzC3fJL+Pg6jQBf6BqP89MBJvrezgecOtRKNOStnT+GDK2dw5zXTKdK9UCQgA9EYB5o72P7qmfMPHWnr6gNgckEOK2dNOR/wy2YUk5ed2fPvCvQM1tE7wFeeOcRTuxo5e26AiqIJvP+6Ku5ZOYOrdH8OGYfcnaOnzrH91cRDSY6e5nBr/OR8bnYWy2cUnw/4BRVFFCSeFpWXnZUR53kU6Bmqpb2Xj//jNupPdnHHsviUytuuLtNlhZJ2TnX1sePoGXYejT+0ZE9DO4OxN2aXGUzIHvq4wKw3Pjpw2KME83KyMIL5t3D9VSXcvGDqZf23V3q3RUlD9Sc7+dj67Zw91883P7Gat80rC7okkctWOimP25ZUcNuSCiA+B197/CxHT3XTMxCldyCW+Bqlpz9Kz0D81Zt439U3SGtnX3x/ok3vCM9mHSvZWXbZgf6mx035ESVwO4+e4VPf2k52Vhb/93duZGlVcdAliaRUfm7k/JOZ5HUK9JB5Zv8JPvvEi0wvzudbn1jNrNKCoEsSkTGiQA+RJ7Yd44+f2sOyqmLWf3wVpZO01FokkyjQQ8Dd+Zuf1vOVnxxizYJyHv3QdUzUU91FMo7+1ae5wWiMP920jye2HeMD183gSx9YFvon24vIyBToaax3IMpnn9jFM/tP8J9vnst/fdeCjLgOV0RGpkBPU2fP9fOpb+3gxWNn+OJdS/jYW6qDLklEAqZAT0ONZ3v42PptHDt1jkc/dB13LpsedEkiMg4o0NNMXUsHH1u/jXP9Ub71ydW6DldEzlOgp5EXDp/iP317BwW5Eb77wI0srCgKuiQRGUcU6Gli655m/suGWmaW5PPtT11P1eT8oEsSkXFGgT7OuTvrf/Eq/+MH+7l25mT+4WOrmDIxN+iyRGQcUqCPYwPRGH+2eR//8utjvGvxNL5277Xk52b2vaBF5MIU6ONUe88AD/7Lizz/chu/c9NVPHTbQj25RUTelAJ9HDp6qptPfnM7x06f46/uuYb/UDMz6JJEJA0ktUbczG43s4NmVm9mD4+w/7+ZWW3itdfMomZWkvpyw2/bkdO879FfcKq7n+986nqFuYgk7aKBbmYR4FHgDmAxcJ+ZLR7axt2/7O4r3H0F8HngOXc/PRoFh9n3dzbw4cdfYEpBLk995q3ccJWuMReR5CUz5bIaqHf3wwBmtgFYC+y/QPv7gCdSU15miMWcv37mII/++yu8ZW4pf//hlRQX6KHNInJpkgn0KuD4kO0G4PqRGppZAXA78OAF9t8P3A8wa9asSyo0rHr6o/zBd2vZuqeF+1bP5L+vXaq7JYrIZUkm0Ee6tOJCT5Z+L/CLC023uPtjwGMQf0h0UhWG2MmOXj797R3saWznT969iE+9bY7uligily2ZQG8Ahp6ZmwE0XaDtvWi6JSn7mtr59Ld20N4zwDc+UsMti6cFXZKIpLlk/t9+OzDPzOaYWS7x0N48vJGZFQM3AZtSW2L4PLP/BB9c9ysAvvfAWxTmIpISFx2hu/ugmT0IPA1EgPXuvs/MHkjsX5doejfwY3fvHrVq05y78/jzR/jLHx7gmqpivvHRGqYWTQi6LBEJCXMPZiq7pqbGd+zYEcj3DsJANMYXNu3liW3HuXNZBX/9wRVaxi8il8zMdrp7zUj7tFJ0jPzBv+5m8+4mPvvOq/ncLfO1jF9EUk6BPgZ+tLeFzbub+Nwt8/m9W+YFXY6IhJQueB5l7T0DfGHTXhZPL+IzN88NuhwRCTGN0EfZ/9p6gFPd/az/+CotGBKRUaWEGUW/rG9jw/bjfPrtc1haVRx0OSIScgr0UdLTH+XhJ/dQXVrA526ZH3Q5IpIBNOUySr7yk0McO32ODfffwIQcXZ4oIqNPI/RRsPv4WR5//jD3rZ6lW+CKyJhRoKdY/2CMh77/EuWFeXz+zoVBlyMiGURTLin2f557hbqWTr7x0RqKJuie5iIydjRCT6H6k5387b/V855rpnOrbrglImNMgZ4isZjz0Pf3UJAX4c/vWhJ0OSKSgRToKfKdF46y8+gZ/vTdiymblBd0OSKSgRToKdBw5hx/9aM63jG/nPdfVxV0OSKSoRToV8jd+eOn9uLAX969VI+QE5HAKNCv0MbaRp471Mof3raAGVMKgi5HRDKYAv0KtHX18cUt+7lu1mQ+cmN10OWISIZToF+BL27Zz7m+KI984BoiemCFiARMgX6ZfrL/BFt2N/HgO69m3rTCoMsREVGgX46O3gH+ZONeFkwr5IGb9NAKERkftPT/MjzywzpOdvay7iMryc3W70QRGR+URpfohcOn+OdfH+OTb53DipmTgy5HROQ8Bfol6B2I8vkn9zCrpIDff5ceWiEi44umXC7B1376MkfauvnnT19PQa7+6ERkfNEIPUmdvQOs//kR3n9tFW+9uizockREfoMCPUlP7ztB32CM/3jj7KBLEREZUVKBbma3m9lBM6s3s4cv0GaNmdWa2T4zey61ZQZvU20js0oKuFYnQkVknLpooJtZBHgUuANYDNxnZouHtZkM/B1wl7svAT44CrUG5mRnL7+ob2PtikrdfEtExq1kRuirgXp3P+zu/cAGYO2wNh8CnnT3YwDufjK1ZQZry+5mYg5rV1QGXYqIyAUlE+hVwPEh2w2Jz4aaD0wxs2fNbKeZfXSkA5nZ/Wa2w8x2tLa2Xl7FAdhc28iSyiKunqol/iIyfiUT6CPNMfiw7WxgJfBu4DbgT83sNy7UdvfH3L3G3WvKy8svudggHGnrZndDO+9boQdXiMj4lszF1A3AzCHbM4CmEdq0uXs30G1mPwOWA4dSUmWANu5qxAzeu1zTLSIyviUzQt8OzDOzOWaWC9wLbB7WZhPwdjPLNrMC4HrgQGpLHXvuzqbaRm68qpSK4glBlyMi8qYuOkJ390EzexB4GogA6919n5k9kNi/zt0PmNmPgJeAGPC4u+8dzcLHwksN7bx66hy/u0Z3VBSR8S+p9evuvhXYOuyzdcO2vwx8OXWlBW9jbSO5kSxuXzo96FJERC5KK0UvYDAaY8vuZt65cCrF+TlBlyMiclEK9Av45SunaOvq433X6mSoiKQHBfoFbKptonBCNmsWTA26FBGRpCjQR9A7EOXpfS3csbSCCTmRoMsREUmKAn0EPzlwgq6+QS0mEpG0okAfwabaJqYW5nH9VaVBlyIikjQF+jBnz/Xz7MGT3LW8kkiW7qwoIulDgT7M1j0tDESd912r6RYRSS8K9GE21jYyt3wiSyqLgi5FROSSKNCHaDrbw7Yjp1m7okoPshCRtKNAH2Lz7vhNJPUgCxFJRwr0ITbuauTaWZOZXTox6FJERC6ZAj3hYEsndS2duvZcRNKWAj1hU20jkSzj3dfozooikp4U6EAs5myqbeJtV5dRNikv6HJERC6LAh3YeewMjWd7dGdFEUlrCnTiJ0Mn5GTxrsUVQZciInLZMj7Q+wdj/GBPM7curmBiXlIPcBIRGZcyPtCff7mVs+cGeJ+uPReRNJfxgb6xtokpBTm8Y3550KWIiFyRjA70rr5Bntnfwp3LppMTyeg/ChEJgYxOsWf2t9A7ENOdFUUkFDI60DfuaqJqcj4rZ00JuhQRkSuWsYHe1tXHz+vbWLuikiw9yEJEQiBjA/0HLzUTjTlrde8WEQmJpALdzG43s4NmVm9mD4+wf42ZtZtZbeL1hdSXmlobaxtZWFHIgorCoEsREUmJi66kMbMI8ChwK9AAbDezze6+f1jT5939PaNQY8odPdXNrmNnefiOhUGXIiKSMsmM0FcD9e5+2N37gQ3A2tEta3Rtqo0/yOKu5VpMJCLhkUygVwHHh2w3JD4b7kYz221mPzSzJSmpbhS4OxtrG1k9p4TKyflBlyMikjLJBPpIl4D4sO0Xgdnuvhz4W2DjiAcyu9/MdpjZjtbW1kurNEX2NXVwuLVbD7IQkdBJJtAbgJlDtmcATUMbuHuHu3cl3m8FcsysbPiB3P0xd69x95ry8mCW2m/c1UhOxLhzme6sKCLhkkygbwfmmdkcM8sF7gU2D21gZhVmZon3qxPHPZXqYq9UNOZseamJm+ZPZXJBbtDliIik1EWvcnH3QTN7EHgaiADr3X2fmT2Q2L8OuAf4XTMbBHqAe919+LRM4F4+2cmJjj7+8DaNzkUkfJK6AXhiGmXrsM/WDXn/deDrqS0t9eqaOwFYNqM44EpERFIvo1aKHmjpIDeSxZyyiUGXIiKSchkV6HXNnVw9dZJulSsioZRRyVbX0sHC6VrqLyLhlDGBfrq7nxMdfSyqKAq6FBGRUZExgV7X0gGgEbqIhFbmBHriCpeFGqGLSEhlTqC3dFA2KZfywrygSxERGRUZFOidGp2LSKhlRKBHY87Blk4W6mEWIhJiGRHor57qpm8wxsLpGqGLSHhlRKC/fkJUI3QRCa/MCPSWDiJZxtVTJwVdiojIqMmIQD/Q3MlVZROZkBMJuhQRkVGTEYEeX/Kv+XMRCbfQB3pH7wANZ3o0fy4ioRf6QD/UEj8hukhL/kUk5EIf6AdatORfRDJD6AO9rrmDognZTC+eEHQpIiKjKvyB3tLJwulFJJ5hLSISWqEO9Fhiyf8inRAVkQwQ6kBvPNtDV9+gLlkUkYwQ6kA/0Jx4qIVG6CKSAUId6HUtnZjB/GkKdBEJv5AHegezSwqYmJcddCkiIqMu3IHerIdaiEjmCG2g9/RHOXKqWw+FFpGMkVSgm9ntZnbQzOrN7OE3abfKzKJmdk/qSrw8h0504q4VoiKSOS4a6GYWAR4F7gAWA/eZ2eILtHsEeDrVRV6Oupb4FS66h4uIZIpkRuirgXp3P+zu/cAGYO0I7T4LfB84mcL6LtuB5k4KciPMnFIQdCkiImMimUCvAo4P2W5IfHaemVUBdwPr3uxAZna/me0wsx2tra2XWuslqWvpYEFFIVlZWvIvIpkhmUAfKRF92PZXgYfcPfpmB3L3x9y9xt1rysvLk63xkrl7/B4umj8XkQySzAXaDcDMIdszgKZhbWqADYkbYJUBd5rZoLtvTEmVl+hERx9nzw1o/lxEMkoygb4dmGdmc4BG4F7gQ0MbuPuc196b2TeB/xdUmAMcaHltyb9G6CKSOS4a6O4+aGYPEr96JQKsd/d9ZvZAYv+bzpsHoa45/lCLBbqHi4hkkKTWxLv7VmDrsM9GDHJ3//iVl3Vl6lo6qJqcT3F+TtCliIiMmVCuFI0v+dfoXEQyS+gCvW8wyiutXVryLyIZJ3SB/srJbgZjrhOiIpJxQhfoWvIvIpkqhIHeSW52FtWlE4MuRURkTIUu0A80dzB/2iSyI6HrmojImwpd6mnJv4hkqlAFeltXH62dfbpkUUQyUqgC/WBLfIXooukaoYtI5glVoB9ofu0eLhqhi0jmCVWg17V0Ul6YR+mkvKBLEREZcyEL9A6NzkUkY4Um0AejMQ6d6NL8uYhkrNAE+qunuukfjGmELiIZKzSBfiBxD3Rdgy4imSo0gV7X0kF2ljF3qpb8i0hmCk+gN3cyt3wSedmRoEsREQlEeAK9pVP3QBeRjBaKQG/vGaDxbI/mz0Uko4Ui0F9b8q8RuohkslAE+vmHWmiELiIZLBSBfqC5k8kFOUwr0pJ/EclcoQj015b8m1nQpYiIBCbtAz0Wcw7qoRYiIukf6MfPnONcf1QPhRaRjJf2ga4l/yIicUkFupndbmYHzazezB4eYf9aM3vJzGrNbIeZvS31pY6srqUDM5g/TSN0Ecls2RdrYGYR4FHgVqAB2G5mm919/5BmPwU2u7ub2TXAvwILR6Pg4eqaO5lTOpH8XC35F5HMlswIfTVQ7+6H3b0f2ACsHdrA3bvc3RObEwFnjNS1dGhBkYgIyQV6FXB8yHZD4rM3MLO7zawO+AHwyZEOZGb3J6ZkdrS2tl5OvW/Q3TfI0dPnNH8uIkJygT7Sxd2/MQJ396fcfSHwPuAvRjqQuz/m7jXuXlNeXn5plY7g0IlO3PVQaBERSC7QG4CZQ7ZnAE0XauzuPwPmmlnZFdZ2UXWJe7josXMiIskF+nZgnpnNMbNc4F5g89AGZna1JZZpmtl1QC5wKtXFDlfX3MGkvGyqJueP9rcSERn3LnqVi7sPmtmDwNNABFjv7vvM7IHE/nXAB4CPmtkA0AP89pCTpKPmQEsnCyoKycrSkn8RkYsGOoC7bwW2Dvts3ZD3jwCPpLa0i9ZEXXMH711eOZbfVkRk3ErblaLN7b109A6yUPPnIiJAGgf66/dA1xUuIiKQxoH+2j1c5ivQRUSANA70upZOZkzJp2hCTtCliIiMC+kb6M0dWiEqIjJEWgZ670CUw23duge6iMgQaRno9Se7iMZcI3QRkSHSMtBfW/KvuyyKiLwuPQO9uYO87CyqSycGXYqIyLiRnoGeWPIf0ZJ/EZHz0jTQO3TLXBGRYdIu0Fs7+2jr6tcJURGRYdIu0F9b8q8ToiIib5R2gZ6fE+GWRdNYpBG6iMgbJHX73PGkprqEx6tLgi5DRGTcSbsRuoiIjEyBLiISEgp0EZGQUKCLiISEAl1EJCQU6CIiIaFAFxEJCQW6iEhImLsH843NWoGjl/mflwFtKSwnHajPmUF9zgxX0ufZ7l4+0o7AAv1KmNkOd68Juo6xpD5nBvU5M4xWnzXlIiISEgp0EZGQSNdAfyzoAgKgPmcG9TkzjEqf03IOXUREflO6jtBFRGQYBbqISEiM60A3s9vN7KCZ1ZvZwyPsNzP7m8T+l8zsuiDqTKUk+vzhRF9fMrNfmtnyIOpMpYv1eUi7VWYWNbN7xrK+0ZBMn81sjZnVmtk+M3turGtMtST+bheb2RYz253o8yeCqDNVzGy9mZ00s70X2J/6/HL3cfkCIsArwFVALrAbWDyszZ3ADwEDbgB+HXTdY9DntwBTEu/vyIQ+D2n3b8BW4J6g6x6Dn/NkYD8wK7E9Nei6x6DPfwQ8knhfDpwGcoOu/Qr6/A7gOmDvBfanPL/G8wh9NVDv7ofdvR/YAKwd1mYt8G2PewGYbGbTx7rQFLpon939l+5+JrH5AjBjjGtMtWR+zgCfBb4PnBzL4kZJMn3+EPCkux8DcPd073cyfXag0MwMmEQ80AfHtszUcfefEe/DhaQ8v8ZzoFcBx4dsNyQ+u9Q26eRS+/Mp4r/h09lF+2xmVcDdwLoxrGs0JfNzng9MMbNnzWynmX10zKobHcn0+evAIqAJ2AP8nrvHxqa8QKQ8v8bzQ6JthM+GX2OZTJt0knR/zOxm4oH+tlGtaPQl0+evAg+5ezQ+eEt7yfQ5G1gJ/BaQD/zKzF5w90OjXdwoSabPtwG1wDuBucAzZva8u3eMdnEBSXl+jedAbwBmDtmeQfw396W2SSdJ9cfMrgEeB+5w91NjVNtoSabPNcCGRJiXAXea2aC7bxybElMu2b/bbe7eDXSb2c+A5UC6Bnoyff4E8CWPTzDXm9kRYCGwbWxKHHMpz6/xPOWyHZhnZnPMLBe4F9g8rM1m4KOJs8U3AO3u3jzWhabQRftsZrOAJ4GPpPFobaiL9tnd57h7tbtXA98DPpPGYQ7J/d3eBLzdzLLNrAC4HjgwxnWmUjJ9Pkb8/0gws2nAAuDwmFY5tlKeX+N2hO7ug2b2IPA08TPk6919n5k9kNi/jvgVD3cC9cA54r/h01aSff4CUAr8XWLEOuhpfKe6JPscKsn02d0PmNmPgJeAGPC4u494+Vs6SPLn/BfAN81sD/HpiIfcPW1vq2tmTwBrgDIzawD+DMiB0csvLf0XEQmJ8TzlIiIil0CBLiISEgp0EZGQUKCLiISEAl1EJCQU6CIiIaFAFxEJif8P1gITpb0QznIAAAAASUVORK5CYII=\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "## 绘制图表\n",
    "thresholds = np.linspace(0,1,21) # 创建21个阈值1\n",
    "accuracies = []\n",
    "for t in thresholds:\n",
    "    acc = accuracy_score(y_test,y_pred>t)\n",
    "    accuracies.append(acc)\n",
    "plt.plot(thresholds,accuracies)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:43:34.056106100Z",
     "start_time": "2023-12-09T01:43:33.499606300Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 4.1.2 虚拟基线\n",
    "<p>虚拟基线就是让模型的预测结果是一个固定值。在本例中,我们可以让模型的预测结果总是客户不会流失</p>"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [
    {
     "data": {
      "text/plain": "array([False, False, False, ..., False, False, False])"
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "size_val = len(y_test) # 获取验证集中的客户数量\n",
    "baseline = np.repeat(False,size_val) # 创建一个只含有False元素的数组\n",
    "baseline"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:43:56.030417300Z",
     "start_time": "2023-12-09T01:43:56.010420500Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [
    {
     "data": {
      "text/plain": "0.7535491198182851"
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(baseline,y_test)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:43:59.664153600Z",
     "start_time": "2023-12-09T01:43:59.653157Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    ">- 模型似乎比基线高6个百分点"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 4.2混淆矩阵"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [],
   "source": [
    "t = 0.5\n",
    "predict_churn = (y_pred >=0.5) # 预测阈值为0.5\n",
    "predict_no_churn = (y_pred<t)\n",
    "\n",
    "actual_churn = (y_test==1) # 获取实际目标值\n",
    "actual_no_churn = (y_test==0)\n",
    "\n",
    "true_positive = (predict_churn & actual_churn).sum() # 计算真正例\n",
    "false_positive = (predict_churn & actual_no_churn).sum() # 计算假正例\n",
    "\n",
    "false_negative = (predict_no_churn & actual_churn).sum() # 计算假反例\n",
    "true_negative = (predict_no_churn & actual_no_churn).sum() # 计算真反例\n",
    "confusion_table = np.array(\n",
    "    [[true_negative,false_positive],[false_negative,true_positive]]\n",
    ")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:45:33.850054800Z",
     "start_time": "2023-12-09T01:45:33.830989500Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [
    {
     "data": {
      "text/plain": "array([[1189,  138],\n       [ 193,  241]], dtype=int64)"
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_table"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:45:34.679706600Z",
     "start_time": "2023-12-09T01:45:34.671698700Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [
    {
     "data": {
      "text/plain": "array([[0.67518455, 0.07836457],\n       [0.10959682, 0.13685406]])"
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_table/confusion_table.sum()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:45:41.946714600Z",
     "start_time": "2023-12-09T01:45:41.938719300Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 4.2.3查准率和查全率\n",
    "<p>查准率:所有预测为正例的样本中，真正的正例样本所占的比例.在本文件中,是指预测会流失的客户中实际流失客户所占的比例。(TP)/(TP+FP)</p>\n",
    "<p>查全率:所有实际为正例的样本中,被正确预测为正例的样本所占的比列.在本文中,是指实际会流失的客户中被正确预测为会流失客户所占的比例</p>"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [
    {
     "data": {
      "text/plain": "0.6358839050131926"
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = true_positive/(true_positive+false_positive) # 计算查全率\n",
    "p"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:45:54.175944200Z",
     "start_time": "2023-12-09T01:45:54.167946400Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [
    {
     "data": {
      "text/plain": "0.5552995391705069"
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = true_positive/(true_positive+false_negative)\n",
    "r"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:45:55.008969600Z",
     "start_time": "2023-12-09T01:45:54.999971Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 4.3 ROC曲线和AUC分数\n",
    "### 4.3.1 真正例率和假正例率\n",
    "<p>假正例率(FPR):所有负例中假正例所占的比率</p>\n",
    "<p>真正例率(TPR):所有正例中真正例所占的比率</p>"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fpr:10.399, tpr:55.530\n"
     ]
    }
   ],
   "source": [
    "fpr = (false_positive)/(false_positive+true_negative)\n",
    "tpr = (true_positive)/(false_negative+true_positive)\n",
    "print(\"fpr:{:.3f}, tpr:{:.3f}\".format(fpr*100,tpr*100))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:46:31.497697300Z",
     "start_time": "2023-12-09T01:46:31.488701800Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 4.3.2 在多个阈值下评估模型"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "outputs": [
    {
     "data": {
      "text/plain": "     threshold   tp    fp   fn    tn\n0         0.00  434  1327    0     0\n1         0.01  432  1180    2   147\n2         0.02  429  1078    5   249\n3         0.03  428   998    6   329\n4         0.04  428   945    6   382\n..         ...  ...   ...  ...   ...\n96        0.96    0     0  434  1327\n97        0.97    0     0  434  1327\n98        0.98    0     0  434  1327\n99        0.99    0     0  434  1327\n100       1.00    0     0  434  1327\n\n[101 rows x 5 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>threshold</th>\n      <th>tp</th>\n      <th>fp</th>\n      <th>fn</th>\n      <th>tn</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.00</td>\n      <td>434</td>\n      <td>1327</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.01</td>\n      <td>432</td>\n      <td>1180</td>\n      <td>2</td>\n      <td>147</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.02</td>\n      <td>429</td>\n      <td>1078</td>\n      <td>5</td>\n      <td>249</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.03</td>\n      <td>428</td>\n      <td>998</td>\n      <td>6</td>\n      <td>329</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.04</td>\n      <td>428</td>\n      <td>945</td>\n      <td>6</td>\n      <td>382</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>96</th>\n      <td>0.96</td>\n      <td>0</td>\n      <td>0</td>\n      <td>434</td>\n      <td>1327</td>\n    </tr>\n    <tr>\n      <th>97</th>\n      <td>0.97</td>\n      <td>0</td>\n      <td>0</td>\n      <td>434</td>\n      <td>1327</td>\n    </tr>\n    <tr>\n      <th>98</th>\n      <td>0.98</td>\n      <td>0</td>\n      <td>0</td>\n      <td>434</td>\n      <td>1327</td>\n    </tr>\n    <tr>\n      <th>99</th>\n      <td>0.99</td>\n      <td>0</td>\n      <td>0</td>\n      <td>434</td>\n      <td>1327</td>\n    </tr>\n    <tr>\n      <th>100</th>\n      <td>1.00</td>\n      <td>0</td>\n      <td>0</td>\n      <td>434</td>\n      <td>1327</td>\n    </tr>\n  </tbody>\n</table>\n<p>101 rows × 5 columns</p>\n</div>"
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = [] #  创建一个用于保存结果的列表\n",
    "thresholds = np.linspace(0,1,101)\n",
    "for t in thresholds:\n",
    "    tp = ((y_pred>=t) & (y_test==1)).sum()\n",
    "    fp = ((y_pred>=t)&(y_test==0)).sum()\n",
    "    tn = ((y_pred<t)&(y_test==0)).sum()\n",
    "    fn = ((y_pred<t)&(y_test==1)).sum()\n",
    "    scores.append((t,tp,fp,fn,tn))\n",
    "df_scores = pd.DataFrame(scores)\n",
    "df_scores.columns = ['threshold','tp','fp','fn','tn']\n",
    "df_scores"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:47:17.272094700Z",
     "start_time": "2023-12-09T01:47:17.106095300Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "outputs": [
    {
     "data": {
      "text/plain": "   threshold   tp    fp  fn   tn       tpr       fpr\n0       0.00  434  1327   0    0  1.000000  1.000000\n1       0.01  432  1180   2  147  0.995392  0.889224\n2       0.02  429  1078   5  249  0.988479  0.812359\n3       0.03  428   998   6  329  0.986175  0.752072\n4       0.04  428   945   6  382  0.986175  0.712133",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>threshold</th>\n      <th>tp</th>\n      <th>fp</th>\n      <th>fn</th>\n      <th>tn</th>\n      <th>tpr</th>\n      <th>fpr</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.00</td>\n      <td>434</td>\n      <td>1327</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.01</td>\n      <td>432</td>\n      <td>1180</td>\n      <td>2</td>\n      <td>147</td>\n      <td>0.995392</td>\n      <td>0.889224</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.02</td>\n      <td>429</td>\n      <td>1078</td>\n      <td>5</td>\n      <td>249</td>\n      <td>0.988479</td>\n      <td>0.812359</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.03</td>\n      <td>428</td>\n      <td>998</td>\n      <td>6</td>\n      <td>329</td>\n      <td>0.986175</td>\n      <td>0.752072</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.04</td>\n      <td>428</td>\n      <td>945</td>\n      <td>6</td>\n      <td>382</td>\n      <td>0.986175</td>\n      <td>0.712133</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_scores['tpr'] = df_scores['tp']/(df_scores['tp']+df_scores['fn'])\n",
    "df_scores['fpr'] = df_scores['fp']/(df_scores['fp']+df_scores['tn'])\n",
    "df_scores.head()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:47:28.592671700Z",
     "start_time": "2023-12-09T01:47:28.235697300Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xUVfrH8c+ZSSeNhIQUEpJQQ0sCAaQaBVaaIqhUG1JEwbarK/52Xd3VLe6uDUURUVQQkUVsiCiiFKUHQughQIBAqKEEQkg7vz9ucAMbyACT3CnP+/XKC2bmZuZ7RR5uzj3nOUprjRBCCOdnMTuAEEII+5CCLoQQLkIKuhBCuAgp6EII4SKkoAshhIvwMOuD69Wrp+Pi4sz6eCGEcErp6enHtNZhVb1mWkGPi4tj3bp1Zn28EEI4JaXU3su9JkMuQgjhIqSgCyGEi5CCLoQQLsK0MXQhhLCHkpIScnNzKSoqMjuKXfn4+NCgQQM8PT1t/h4p6EIIp5abm0tAQABxcXEopcyOYxdaa44fP05ubi7x8fE2f1+1Qy5KqfeVUkeUUpsv87pSSk1SSmUrpTKVUm2vIrcQQlyXoqIiQkNDXaaYAyilCA0NveqfOmwZQ/8A6H2F1/sATSq+xgJvX1UCIYS4Tq5UzC+4lnOqtqBrrZcB+Vc4ZADwkTasAoKVUpFXncRGeTvSWTX1ETbuzKG8XFr/CiHEBfYYQ48G9ld6nFvxXN6lByqlxmJcxRMbG3tNH7Z/91ZuOPgRA95vzIE6Lbi5eTgjOjYkKSb4mt5PCCGux/Hjx+nRowcAhw4dwmq1EhZmLOTcuHEjSUlJlJaWkpiYyIcffoifnx9Wq5XWrVtTWlpKfHw8M2bMIDj4+muYPaYtVvVzQZWXzlrrqVrrVK116oUTvlod2qUC8GxnHzo3qseCTYcYMPkXBk9ZycLNeRwpKEI27RBC1JbQ0FAyMjLIyMhg3LhxPPHEE78+rlOnDhkZGWzevBkvLy+mTJkCgK+v76/Ph4SEMHnyZLtksccVei4QU+lxA+CgHd63anXjAEgNPEXqrSkUFJXw6dr9TP8lh3Ez1wMQ4O1BbKgf3h7Gv1dWi6JtbF16JNanbWwwHlaZfi+EqF3dunUjMzPzf57v1KlTlc9fC3sU9K+ACUqp2UBH4JTW+n+GW+zG0xcCoiB/NwABPp6M7pbA/Z3jWLU7n+wjBew+dpZ9+YWUVYyxFxaX8f4ve3hn2W6C/TxJaxpGj8T63NgsjEAf2+d4CiEc25+/3sLWg6ft+p4togJ57taW1/UepaWlfPvtt/TuffH8krKyMhYvXsyoUaOu6/0vqLagK6U+AdKAekqpXOA5wBNAaz0FWAD0BbKBQmCkXZJdSUgC5O+56CkPq4WuTerRtUm9Kr+loKiE5TuP8cO2wyzZcZQvMg7iYVG0jwuhR2I4PRPrE1evTo1HF0K4j3PnzpGcnAwYV+gXCveF53NycmjXrh29evWyy+dVW9C11sOqeV0D4+2SxlYhcZD1/VV9S4CPJ31bR9K3dSRl5ZqM/Sf4YdsRFm87zIvfbOPFb7bRrmFdxnSLp1eLCKwW15sGJYSru94raXu7MFZ+uedPnTpF//79mTx5Mo8++uh1f55zrhQNSYCzR+D8GfD2v+pvt1oU7RqG0K5hCE/3bs7+/EIWbj7EhyuNcfiGoX4MSmlAj8RwWkYFuuQcVyGE+YKCgpg0aRIDBgzgoYceuqpl/lVxzoJet2Ip7Ik9ENH6ut8uJsSPMd0TGNklju+2HOaDFXt4bXEWr/6QRWSQD8kxwSSE1SG+nj+BPv/9T9YiKpAGdf2u+/OFEO4rJSWFpKQkZs+ezT333HNd7+WcBT0kwfg1f7ddCvoFHlYL/dpE0q9NJEcLzvPTjiMs2XGE7XkFLNp6mNJLFjJZFPRpFcnobvGkxNa1Ww4hhHN6/vnnL3p85syZKo+79Pmvv/7aLp/vpAW94gr9khuj9hQW4M3g1BgGpxozMkvKysk9cY6z50sBKCvXLNicx6zV+/hmUx4N6vqSEOZPQr063NgsjLSmYTJUI4SoVc5Z0H2CwC/016mLtcHTaiH+klkwSTHBPHJzEz5LzyV97wl2HztDek4+H6zI4aZmYTx3a0uZOSOEqDXOWdDBGHY5UXNX6Lby9/bgvs5x3Nc5DjCu5D/4JYfXfsjiN68uY9yNCTyU1hhfL6u5QYUQLs95l0xWMRfdEXhaLYzpnsCPT6bRp3UEk37MpucrS/l+yyFpSSCEqFHOW9DrxsOpXCg9b3aSKtUP9OH1oSnMHnsD/t4ejJ2RzpiP0sk/W2x2NCGEi3Legh6SAGg4sdfsJFd0Q0Io8x/tyh/6JrIs6yj9Ji1nXc6VuhELIcS1ceKCfmGmS+3dGL1WF4Zh5j3cGS8PC0OmrmLqsl0yBCOEi7BarSQnJ//6lZOTw5IlSwgKCiIlJYXExET+/Oc/A1z0fPPmzXnyySftlsO5b4qCQ9wYtVWr6CDmP9KVpz/L5G8LtrM//xzP39ZS2gwI4eSqWuKfk5NDt27dmD9/PmfPniU5OZn+/fsD/Pr8uXPnSElJYeDAgXTp0uW6czjvFbpfKHgHOsUVemUBPp68OawtD3ZPYMaqvYz/eD1FJWVmxxJC1KA6derQrl07du3addHzvr6+JCcnc+DAAbt8jvNeoStl9EZ3soIOYLEonumbSHigDy/M38rtk3/htuQoeibWp0m4vyxIEuJafTsRDm2y73tGtIY+/7jiIZW7KsbHx/P5559f9Prx48dZtWoVzz77LEePHv31+RMnTrBz5066d+9ul6jOW9DBGHax9x9eLRrVNZ6oIB/e/Cmbfy7cwT8X7qBNgyBmjOpIkK/0aRfCWVyuq+Ly5ctJSUnBYrEwceJEWrZsyZIlS1i+fDlt2rRhx44dTJw4kYiICLvkcPKCHg/b50NZKVid81T6tI6kT+tIDp0qYuHmPP66YBsTZq1n+v3tZWclIa5WNVfSte3CWPnlns/KyqJr164MHDjw1yv86+HcFSMkAcpL4XSu2UmuW0SQD/d3ieevt7dm+c5jvDB/q9mRhBA1rGnTpjzzzDO89NJLdnk/5y7o9Zoavx7ZZm4OOxrcPoYx3eL5cOVeZqzMMTuOEKKGjRs3jmXLlrFnz/XP2HPOcYoL6rcCFBzMgGZ9zE5jNxP7JLLr6Fn+/PVWujYJ+5+mYEIIx1JVm9y0tDTS0tKqfd7X19dus1yc+wrd2x/CmkHe/96McGZWi+Ifd7TG28PCS99uNzuOEMJJOHdBB4hMhoMbzE5hd+EBPoy7sRELtxxirbQKEELYwPkLelQynDkMp/PMTmJ3o7slEBHow4vfbKO8XNoECHE5rthG41rOyfkLemTFVB8XG3YB8PWy8uQtzdi4/yRfZx40O44QDsnHx4fjx4+7VFHXWnP8+HF8fHyu6vuc+6YoGKu4lMXlboxeMCglmvd/3sML87ex73ghNyeG0yIyUFaTClGhQYMG5ObmXrQC0xX4+PjQoEGDq/oe5y/o3v7G9EUXHEcHo03AP+9swx++2MzLi7J4eVEWUUE+3JwYTo/m9enUKBQfT9kNSbgvT09P4uPjzY7hEJy/oIMx7LL7J7NT1JhW0UF8Ob4LRwqKWLL9KD9sO8y89QeYuWofdbysDG4fwwNd4okJ8TM7qhDCRK5R0KNSIHO2cWM0MNLsNDUmPMCHwe1jGNw+hqKSMlbtPs4XGw4wY+VePlyRQ59WkYzuFk9KbF2zowohTOAiBb3SjVEXLuiV+XhaSWsWTlqzcJ7u05wPVuQwa/U+vtmUR2rDuozulkCvFvWl17oQbsT5Z7lApRujrjmOXp3IIF+e6ZPIymd68Kf+LTh0uohxM9O5+eUlfLQyh8LiUrMjCiFqgTJrqk9qaqpet26d/d5wckcIbggj5tjvPZ1UaVk53289zLvLd7Nh30mCfD3pUXETtXvTegT4SGteIZyVUipda51a1WuuMeQCxjj6rh9Ba2PzCzfmYbXQt3UkfVtHkr43n5mr9rF42xHmrT+Ap1Vxa1IUY7olkBgZaHZUIYQduU5Bj0yGjZ9AQR4ERpmdxmG0axhCu4YhlJaVs2H/SeZvPMh/0nOZt/4AXRvX46lbmpEUE2x2TCGEHdg0hq6U6q2U2qGUylZKTazi9SCl1NdKqY1KqS1KqZH2j1qN6LbGr7lra/2jnYGH1UL7uBD+PKAVKyf24Onezdl+qIDb3/qFZ+Zt4sTZYrMjCiGuU7UFXSllBSYDfYAWwDClVItLDhsPbNVaJwFpwMtKKS87Z72yyGTwrAN7ltfqxzqjID9PHkprxE9P3sioLvHMWbefm15ewo/bD5sdTQhxHWy5Qu8AZGutd2uti4HZwIBLjtFAgDLWo/sD+UDtTq3w8ILYGyBHCrqtAnw8+WP/Fix4tBvRwb6M+SidOWv3mx1LCHGNbCno0UDlv+W5Fc9V9iaQCBwENgGPaa3LL30jpdRYpdQ6pdS6Gum7EN8Njm6HArnSvBrNIgL49MFOdG4Uyu8/y+SNxTtdqtGREO7CloJe1ZSRS/+23wJkAFFAMvCmUup/plBoradqrVO11qlhYWFXHbZa8d2NX+Uq/ar5e3vw3n3tGZgSzcuLsnj2y82UScteIZyKLQU9F4ip9LgBxpV4ZSOBedqQDewBmtsn4lWISALvQCno18jLw8LLdyXx4I0JzFy1j/Efr6eopMzsWEIIG9lS0NcCTZRS8RU3OocCX11yzD6gB4BSqj7QDNhtz6A2sXpAwy6wZ1mtf7SrsFgUz/RJ5Nn+LVi45RD3vreGU4UlZscSQtig2oKutS4FJgDfAduAOVrrLUqpcUqpcRWHvQB0VkptAhYDT2utj9VU6CuK7wb5u+GUfTZddVejusbzxrAUNuw/wb3T13DmvLQPEMLR2bSwSGu9AFhwyXNTKv3+IPAb+0a7RpXH0ZOGmpvFyd2aFIWPp5VxM9N54IO1fDiyA75e0ntdCEflGs25KgtvCb4hMuxiJ71a1OfVIcmszcnnwZnpnC+VMXUhHJXrFXSLBeIqxtFl6p1d3JYUxUuD2rAs6ygj3l1N3qlzZkcSQlTB9Qo6QPyNcGo/nMgxO4nLGNw+hknDUtiWd5q+ry/npx1HzI4khLiEaxb0hJuMX7fPNzeHi7ktKYqvH+lKRJAvI6evZfSHa5m9Zh9HTheZHU0IgSv1Q7/UtJ5QdBrGr3b7drr2VlRSxuuLd/JVxkEOnDSGX7o0DmV0twTSmoah5L+3EDXmSv3QXbegp38AXz8Go36AmPY19zluTGvNjsMFfL/lMLNW7+PQ6SKahPsz4ebG3JYUJYVdiBpwpYLumkMuAC0HgacfbPjI7CQuSylF84hAHu3RhGW/v4lXhyRhtSgem53B0Kmr2HGowOyIQrgV1y3oPoHQciBsngfFZ81O4/K8PCwMTGnAN492428DW7PjcAF9Jy3nL19v5XSRrDQVoja4bkEHSLkbis/Ali/MTuI2rBbF8I6x/PS7NAanxjB9xR56vLyUzzfkSgdHIWqYaxf02E4Q2hg2zDA7idupW8eLvw9qzRcPdyEqyIcnPt3IAx+s5ay0EBCixrh2QVfKuErftxKO7TQ7jVtKignm84e78PytLVi28xjD3l3FsTPnzY4lhEty7YIOkDQMlBUyPjY7iduyWBT3d4ln6j3tyDpcwB1vryDnmNzXEMLeXL+gB0RA456wcTaUSx8SM/VIrM+sMTdw+lwJw99dxZECWZAkhD25fkEHSBkBBXmw60ezk7i9trF1mTGqIycKSxj7UbpsoCGEHblHQW/ax+jAuGGm2UkE0Co6iFeHJJGx/yRPf5Yps1+EsBP3KOgeXtBmMOxYAIX5ZqcRQO9WkTx1SzO+zDjIi99sk7nqQtiBexR0gOQRUFYMm+aanURUeDitEUPbx/Dez3vo/PcfeXH+VvYdLzQ7lhBOy3V7uVRlSldjxsuDS2v3c8UVZeaeZNryPXyzKY+yck3ziAB6JIYzIDmapvUDzI4nhENxz14uVUm+G/Iy4NBms5OISto0CGbSsBSW/f4m/tA3kSBfT6Ys3c1tb/5M1mHpByOErdyroLcZDB4+sOYds5OIKkQH+zKmewKfPtiJ5b+/CX9vDybMWs+5YpkJI4Qt3Kug+4UYY+kbZ0PBIbPTiCuICvbl1SHJ7Dxyhj9/vcXsOEI4Bfcq6ACdH4HyUlj1ltlJRDW6NQnj4bRGzF67ny8zDpgdRwiH534FPSQeWtwO66ZD0Smz04hqPNGzKakN6/L4pxk8OGMd63LyZd66EJfhfgUdoOvjcP40rHvf7CSiGh5WC+/d157xaY1ZvSefO6esZOBbK/gmM4/SsnKz4wnhUNxr2mJlH90OR7bCY5ng6WNeDmGzwuJSPkvP5b2f95BzvJAGdX0Z2z2BER0bYrXIdnfCPci0xap0eQzOHIbNstDIWfh5eXBPpzgW/y6Nd+5pR2SQD3/6cgv3vb+GowXSklcI9y3oCWlQrxmsl80vnI3VorilZQRzHuzES3e0Zm1OPn0nLWdF9jGzowlhKvct6EpB8nDYvwqOZZudRlwDpRRD2sfy5YQuBPp4MHzaaiZ+lkn+2WKzowlhCvct6ABJQ2XzCxfQPCKQryZ0ZUy3eOam53LTv5fw8eq9MhtGuB33Luiy+YXLqOPtwR/6teDbx7rRIjKQP3y+mZe/zzI7lhC1yqaCrpTqrZTaoZTKVkpNvMwxaUqpDKXUFqWU83S/ShkBBQdh109mJxF20KR+ALPGdGRYhxje/CmbyT/JcJpwHx7VHaCUsgKTgV5ALrBWKfWV1nprpWOCgbeA3lrrfUqp8JoKbHcXNr/ImAlNepqdRtiBUooXb2/NueIy/vXdDnw9rTzQNd7sWELUOFuu0DsA2Vrr3VrrYmA2MOCSY4YD87TW+wC01kfsG7MGXdj8Yvs3svmFC7FaFP++K4lbWtbnL/O3MnjKSr7fcoiychlXF67LloIeDeyv9Di34rnKmgJ1lVJLlFLpSql7q3ojpdRYpdQ6pdS6o0ePXlvimiCbX7gkD6uFN4a15Y/9Ejlw8hxjZ6TT65WlZB+RlrzCNdlS0KtagnfpZY4H0A7oB9wCPKuUavo/36T1VK11qtY6NSws7KrD1pjINhCZDOnTQWZGuBQvDwujuyWw9Kk03hyeQsH5Uoa/u5q9x8+aHU0Iu7OloOcCMZUeNwAOVnHMQq31Wa31MWAZkGSfiLWk/SijFcD+1WYnETXAw2qhf5soZo7qSElZOcPfXc3Bk+fMjiWEXdlS0NcCTZRS8UopL2Ao8NUlx3wJdFNKeSil/ICOwDb7Rq1hre4A7yBY+57ZSUQNahYRwEcPdOT0uRJGTFvNoVNFZkcSwm6qLeha61JgAvAdRpGeo7XeopQap5QaV3HMNmAhkAmsAaZprZ1rnzevOsZCo61fwFlZQu7KWjcIYvrI9hwtOM8db6+QMXXhMty322JVjmyDt26AXn8xmncJl7b5wCnun76WkrJy3r8/lXYNQ8yOJES1pNuircIToWEXY/OLcum17epaRQcx76HOhNTxYvi7q5m3PtfsSEJcFynol0p9AE7sgd0/mp1E1ILYUD/mjutEUoNgfjtnI7+fu1E2pRZOSwr6pRJvrVg5OsvsJKKWhPp7M2tMRybc1Jj/pOcyYPLP5ByTaY3C+UhBv5SHN7QYADu+hWL5S+0uPKwWnrylGR+O7PDrzdLM3JNmxxLiqkhBr0rrO6Gk0Cjqwq10bxrG3Ic64+tlZejUVSzNcqAVzUJUQwp6VWI7Q0AkbP7M7CTCBI3C/Jn3UGcahtZh1AdrefSTDXyZcYBThSVmRxPiiqSgV8VigZaDYOciOHfC7DTCBOGBPnz64A3clRrDL9nHeGx2Bm1fXMTfF2yjpExmQAnHJAX9clrfAeUlsG2+2UmESQJ9PPn7oNas+UNP5j3cmUEp0byzbDdD3lnJAWkbIByQFPTLiWoLdeNl2EVgtSjaxtblX3cl8cawFLIOn6HfpOUs3ynj68KxSEG/HKWM/i57lsIZ52nvLmrWrUlRfP1IVyICfRg5fS1fbDhgdiQhfiUF/Upa3wm6HDbMMDuJcCDx9eowZ1wn2seF8PinGby7bLfZkYQApKBfWXgiNO4Fi1+ADTPNTiMcSKCPJx880J5+bSL564JtTF22y+xIQkhBr9bgjyAhDb4cb/R4EaKCt4eVN4am0LtlBP/+Losdh6RrozCXFPTqePnBsNnQ5Dcw/3FpCSAuYrEo/jqwFQE+Hjw1dyOlMqVRmEgKui08fWDITKMT48KJ0i9dXCTU35sXbm9FZu4p3pHxdGEiKei28vCGfq/A+TPw44tmpxEOpm/rSPq1ieS1H7LYfOCU2XGEm5KCfjXCm0OHsZD+AeRtNDuNcDAvDGhFsJ8Xg95awcvf75A2vKLWSUG/WmkTwS8Evn0aTNrtSTimkDpefPNIV/q1ieSNH7Pp+cpSft4pw3Oi9khBv1q+wdDjT7BvJWyaa3Ya4WDCA314dUgyn469AV8vK/dPXyM7IYlaIwX9WqTcY7QGWPAknNhrdhrhgDomhPL5w53pEB/Cb+dsZMrSXZi1f69wH1LQr4XFCne+Z6winTsSSovNTiQcUICPJ9NHtufWpCj+8e12/vDFZopKZFxd1Bwp6NcqJAEGTIYD6bDoT2anEQ7K28PK60OSGXdjI2at3scdb6+Q7e1EjZGCfj1a3AYdH4LVb8OWL8xOIxyUxaKY2Kc50+5NJffEOfq/8TM/bj9sdizhgqSgX69ef4HoVKM1wOGtZqcRDqxni/oseKwbMSF+/G7ORk6dkx2QhH1JQb9eHl7GKlIvf/hkKBTmm51IOLDoYF/+dWcbTp4r4a2fss2OI1yMFHR7CIyEoR9DwSGYcy+UyZWXuLxW0UEMTIlm+i857M8vNDuOcCFS0O2lQSrc+jrkLIdvfy+LjsQVPXVLMywW+Od3O8yOIlyIFHR7Sh4GXR6Hde/Dkr+bnUY4sMggX8Z0S+DrjQfZsE82Ihf2IQXd3no+Dyl3w9KXYOVbZqcRDuzBGxtRz9+bhz9ez8LNebLwSFw3Kej2phT0fx0Sb4XvnpH+6eKy/L09eO++VIJ8PRk3cz33vr+GPTJHXVwHKeg1weoBd7xXsdPRBNg23+xEwkElxQQz/5GuPHdrCzL2neTOt1eQe0JulIprY1NBV0r1VkrtUEplK6UmXuG49kqpMqXUnfaL6KQ8vGHIxxCVYrQH2LPM7ETCQXlYLYzsEs/n47tQXFbO6A/XceZ8qdmxhBOqtqArpazAZKAP0AIYppRqcZnjXgK+s3dIp+XtDyP+Y7QJ+GSY0SZAiMtoHO7PWyPasvPIGR77ZANl5TKmLq6OLVfoHYBsrfVurXUxMBsYUMVxjwCfAUfsmM/5+YXAPZ8bv07vCz+/KvPUxWV1axLG87e2YPH2I/xLpjSKq2RLQY8G9ld6nFvx3K+UUtHAQGDKld5IKTVWKbVOKbXu6NGjV5vVeQVGwQPfQ+Oe8MPzMPUmOJhhdirhoO7pFMewDrFMWbqL1buPmx1HOBFbCrqq4rlLfxZ8DXhaa33F3qBa66la61StdWpYWJitGV3DhdWkQ2bC2aPwQX/I32N2KuGgnu2fSMNQP56am0lhsYynC9vYUtBzgZhKjxsABy85JhWYrZTKAe4E3lJK3W6XhK4m8VYYvQgsFvhslAy/iCr5eXnwzzvasC+/kJe+3W52HOEkbCnoa4EmSql4pZQXMBT4qvIBWut4rXWc1joOmAs8rLWWfrKXExwLt71h3CT98QWz0wgH1TEhlPs7x/Hhyr2s3CVDL6J61RZ0rXUpMAFj9so2YI7WeotSapxSalxNB3RZLQZAu5Hwy+uQvdjsNMJB/b53MxqG+vHbORkcOHnO7DjCwSmzlhunpqbqdevWmfLZDqPknHGDtCAPhsyA+O5mJxIOaMvBUwyduorQOl7MebAT4YE+ZkcSJlJKpWutU6t6TVaKmsnTF4bPhoAImDHQaOolxCVaRgXxwcgOHCk4z93vrSb/rOxhK6omBd1sdeNg1CJIuAnmPwHfPg3l5WanEg6mXcO6TLsvlb3HC7l72mqOFpw3O5JwQFLQHYFPIAz/FG4YD6unwLzRUCpXYeJinRvVY+q9qew5dlY2mxZVkoLuKCxW6P03o/3u5s9g9nAoliZN4mI3Ng1j1piOFBSVcMfbK8jMPWl2JOFApKA7mq5PGDsfZf8AMwfB+QKzEwkHkxJbl7kPdcbXy8qwqavI2C9FXRikoDuidvfDne/B/jUwawgUy4/W4mKNwvyZO64zof7e3PvearYePG12JOEApKA7qlZ3wKCpsG+l0amxROYgi4tFBPnw8eiO1PH24J73VpN95IzZkYTJpKA7stZ3woC3jF7qn94N5+UvrLhYTIgfM0d3RCkYMW0V+47LfRd3JgXd0SUPg9smwa4fYXpvOJVrdiLhYBqF+TNzdEfOl5Yz4r1V5J2Sn+bclRR0Z9D2Xhg+B07sNVaW7l9rdiLhYJpHBPLhyA6cOFvCiGmrOXZG5qm7IynozqJJLxj9A3jVgQ/6wcZPzU4kHExSTDDv39+egyfPMXjKSqYu28Wuo2cwq72HqH1S0J1JWDMY8yPEdIDPxxqbZciqUlFJh/gQ3r+vPV4eFv62YDs9Xl5K30k/S7sANyEF3dlc2NKu3f3GdnafjoDCfLNTCQfSuXE9Fj7enV8m3szzt7Yg+0gBz321xexYohZIQXdGVk/o/xr0fgl2fg9vdzYWIglRSXSwL/d3iefRm5vw9caDfLspz+xIooZJQXdWSsEN44xxdZ8gmHmH0dyrVG6GiYuNS2tEq+hA/vjFZo7LzVKXJgXd2UWlwNil0GmC0X531mCZry4u4mm18O+7kjhdVMKfZOjFpUlBdwWePnDLX/+7COmjATKuLi7SPCKQx3s25ZvMPD5Ll7UMrkoKuitJGQGDZ8ChTJjeF/L3mJ1IOJBxNzaiY3wIz365WdoEuCgp6K4msT+M+A8UHIR3uhuteIUArBbF60NT8PG0MmHWeopKysyOJOxMCrorSkiDcT9DWHOY+wB8OQHOHDU7ldbQWUIAABFiSURBVHAAEUE+vDw4ie2HCnhh/laz4wg7k4LuqoJjYeQC6PY72DATXm0JX46Hw3JTzN3d1CycB29M4OPV+1ix65jZcYQdSUF3ZVZP6PEnGL/GGF/f9JkxZ/2L8XLT1M090bMpEYE+vPJ9lrQGcCFS0N1BWFPo/yr8dit0eRw2fgKTO8CmuSB/md2Sj6eV8Tc3Zt3eEyzNkuE4VyEF3Z34hUCvP8ODSyEoBj4bBW+2h18myRi7GxqSGkN0sC+vLJKrdFchBd0dRbQ2Vpje/jb4hcKiZ+GVRPj+j7IxtRvx8rDwWM8mZOaeYtHWw2bHEXYgBd1dWayQPBxGfQcPr4akIbDiDXjrBmMzDeEWBqVEE1+vDq8syqK8XK7SnZ0UdAHhzWHAZLj/G+NG6oyBMK0nrHobTktDJ1fmYbXweM8mbD9UwJNzN1JSJu2YnZkUdPFfcV1h3C/wmxehpAgWTjSGYmYNgd1L5Aaqi7otKYrf9mrKvPUHGP3hOs6eLzU7krhGyqybIampqXrdunWmfLaw0dEs2DQH1k2HwmMQ3hLSJkLirUa3R+FSZq/Zx/99vonW0UF8NKojQb6eZkcSVVBKpWutU6t6Ta7QxeWFNYWb/whPbDGGZHQZzLnHaP51ZLvZ6YSdDe0Qyzv3pLLl4GmenpspM1+ckE0FXSnVWym1QymVrZSaWMXrI5RSmRVfK5RSSfaPKkzj6QMpdxvDMX3/DXkbjQVKs4YYUx4PpENZidkphR30alGfiX2as3DLIT5audfsOOIqeVR3gFLKCkwGegG5wFql1Fda68qNIPYAN2qtTyil+gBTgY41EViYyOoBHcZAy0Gw/GXY+R1kLax4zRvqt4TIJIjtBI17Qp1Qc/OKazKqazwrdh3nr99so13DurSKDjI7krBRtWPoSqlOwPNa61sqHj8DoLX++2WOrwts1lpHX+l9ZQzdRRQcgr0r4OB648o9byMUnQJlgQbtoVlfaDXI6C0jnEb+2WL6vr4cb08L8x/pSoCPjKc7iiuNodtS0O8EemutR1c8vgfoqLWecJnjnwSaXzj+ktfGAmMBYmNj2+3dKz/SuZzycsjbAFnfG1fveRnG8w3aQ/N+0LArRCUb0yOFQ1uzJ5+hU1fSr00Uk4Ymo+RGuEO4UkGvdsgFqOpPscp/BZRSNwGjgK5Vva61nooxHENqaqrccXFFFgtEtzO+bnrG2GRjy+ewZR788LxxjKcftLoD+v4LPH1NjSsur0N8CL/t1ZR/f59Fl0ahDO0gP2U5OlsKei4QU+lxA+DgpQcppdoA04A+Wuvj9oknnF5IPHT7rfF15ijs/cVYibr+I6OV79BZEBhpdkpxGQ+lNWbV7nye+2oLybHBNI8INDuSuAJbZrmsBZoopeKVUl7AUOCrygcopWKBecA9Wuss+8cULsE/DFreDrdNMgr50R3w7k2wf63ZycRlWC2KV4YkEeDjyfiP11NYLIuOHFm1BV1rXQpMAL4DtgFztNZblFLjlFLjKg77ExAKvKWUylBKyd1OcWXN+8Ko78HiAe/1NOa2Z31vjMELhxIe4MNrQ5LZdfQsH6/aZ3YccQWyUlSY69xJWPc+rHnX2Ae1bpwxvt7qDghvIStSHcgdb6/gRGExi397o9wgNZGsFBWOyzfYGF9/PBMGTYO68fDza8bCpXe6w85F0kPGQQxtH8Puo2dZm3PC7CjiMqSgC8dg9YQ2d8G9X8DvdhgrUs8XwMd3wgf9Yd9qKewm69cmkgBvD2avlWEXRyUFXTge/zBjRer4NUZhP7YD3v8NTOkGq6fCOblCNIOflwe3JUexYFMep85JqwdHJAVdOC4PL6OwP5oB/V4xNuX49il4uTl8NgZyfpar9lo2rEMsRSXlfJVxwOwoogpyU1Q4l7yNsH4GZM6B86eMm6jN+kKT30DDLsY/AqJG9Zu0HK3hm0e7ys1RE8hNUeE6IpOg37/hd9th4FQIbQJr34MZt8NLcfDR7bD0X5ArFws1ZWiHWLbmneanHUfMjiIuIVfowvkVF8KeZbBrMeT8Ake2GM8n3ma0FwiIMDefiykoKqHfpJ/Zl1/IoJRoJvZtTniAj9mx3MZ1NeeqKVLQRY0pzDfmti/9J3j4QK/nIeVeo/2vsIvC4lIm/5TNu8v24O1hYfKItnRvGmZ2LLcgQy7CvfiFQPcn4eGVENkG5j8Bk9vDho9lIw478fPy4KlbmrPw8W5EBfvy2OwN5J06Z3YstycFXbiu0EZw39cwZCZ4+cOXD8Mb7Yw9UkuLzU7nEhLC/Hnr7racLy3nsU8yKC2T1g1mkoIuXJtSxqbWDy6D4XOgTj2Y/zhMSq6Y037S7IROr1GYP38b2Jo1Ofm8vnin2XHcmhR04R6Ugqa3wOjFcPc8Ywelb5+CfzeBT4ZB5n+kuF+H21OiGZzagDd/ymZF9jGz47gtuSkq3JPWcGA9bP7M2ICj4CAoq7EfatNbIHm4cTUvbHauuIy+k5ajtWbh493x8bSaHcklyU1RIS6lFDRoB73/Bk9sgQe+gy6PGW0FFj0Lr7WG7/4ABYfNTuo0fL2s/GVAS3KOFzJ12W6z47gluUIX4lJHd8Dyl2HTf8Diadxc9a8PgVHQrI+xMtUiV5+XM37WehZtPcyiJ7rTMLSO2XFcjlyhC3E1wprBoKkwYZ3RSyYkAc6fhqzv4NO7YVIKrHwLik6bndQhPduvBZ4WxfNfbcGsC0Z3JVfoQtiqrBR2fGMU8/2rwCsA2t4LHccaPWXEr6Yt382L32xj8vC29Gsje8bak6wUFcLeDqQbhX3rF6DLIbghBERCQH1jBk1oY+MrKgU8fc1OW+tKysq54+0V7Dpyhs8e7iybS9uRFHQhasqpA7BhBhzLMm6gFuTByX1QXrEi1T8C0iZCyj1u13rg0KkibnvzZ7w8LHwxvgv1/L3NjuQSpKALUZvKy+DUfji8BX55HfavNrpCdnnMWOTkG2x2wlqTmXuSu6aspHV0EB+P6Yi3h9xMvl5S0IUwi9awYwEsfgGObgOrFzTqAU16QWQy1G8Jnq7dqXB+5kEmzNpAdLAvPRLD6ZFYn86NQvG0ypyMayEFXQizaQ0H18PmecZCptMVO/4oK0S0goZdoWFn48svxNysNWDh5kPMTd/Pz9nHKCopJzkmmPfvb09IHdmQ5GpJQRfCkWhtjLPnbYS8DNi/BnLXQmmR8Xp4S4jrAvHdISENvAPMTGtXRSVlzM/M4/8+30SDYF8+fKADMSF+ZsdyKlLQhXB0peeNmTN7fzE26di/GkoKjYVNDTsbi5laDIBA15gCuDYnn1EfrMXb08r0+9vTKjrI7EhOQwq6EM6mrMQo6lnfGV/HdgDK2De1xW0Q1xXCEsHivOPQWYcLuO/9NeSfLea5W1syrEOM7FFqAynoQji7o1mwZZ7RTOxYlvGcbwgk3AjJd0Ojm5yyHcGxM+d54tMMlu88xm1JUfxtUGv8vd1reufVkoIuhKvQGk7uNYZl9v4CWQuh8DgExUCrQcZVe2hjCIkHv1CjCZmDKy/XvL10Fy9/v4O6fl7cfUND7unUUOatX4YUdCFcVWmxMS1y/Yewe4mxavUCi6exQXZApNGPJrQxhDeHxj0dcvXqhn0nmPxTNj9sO4KXh4Ux3eL5Xa9mWCyO/49SbZKCLoQ7KC2GEzmQv8v4teAQnDkMp3Ihf/d/p0r6BEGbIUbP9/CW4OFYUwezj5xh8k/ZfL7hAP3bRPLy4CRZkFTJlQq6DFYJ4So8vCCsqfFVleJCY3rkhhmQ/iGsmWrMgw+OgZBGxgwa/wijTXBcN6jXxJQhm8bh/rwyOInmEQH8/dvtHD9TzDv3tiPQx7PWszgbuUIXwh0V5sPORXB8JxzPhvw9xhX92SP/HbapGwdNfgMRrf/bbMw/vFZjfr4hl6f+k4mHVREXWodGYf60bViXu1IbuG2Bv+4hF6VUb+B1wApM01r/45LXVcXrfYFC4H6t9forvacUdCEcUHmZsegp+wej4O9ZBqXn/vt6RBtodQe0vB0CooznlAJrzRXX9L35LNh0iD3HzrLr6Bn2Hi+kjpeVIe1jua9zQ7fbROO6CrpSygpkAb2AXGAtMExrvbXSMX2BRzAKekfgda11xyu9rxR0IZzAhUZjx7ONZmNbv4IDVfy9DWwAkUnGV1izipk2CeBl/1Wgmw+cYtry3czPzKO0XNO0vj89EuvTtXE9Gof7Ex7g7dLz2a+3oHcCntda31Lx+BkArfXfKx3zDrBEa/1JxeMdQJrWOu9y7ysFXQgndSLHWOx0vsB4XF5mzI0/lAnHdgKVaopnHfD2By9/u1/Fl5SVc+Z8KWfPl1JYXPbr8xal8LA6dkE/1Ogubhjx3DV97/XeFI0G9ld6nItxFV7dMdHARQVdKTUWGAsQGxtrw0cLIRxO3Tjo+GDVrxWfheO7Ksbld0HhCSg+Y3yVl9o1hidQt+KrpExz8lwxhefLOHO+lNOlZdV8t7k8AurXzPvacExV/9RdellvyzForacCU8G4Qrfhs4UQzsSrDkS2Mb5qkScQVquf6JhsaQSRC8RUetwAOHgNxwghhKhBthT0tUATpVS8UsoLGAp8dckxXwH3KsMNwKkrjZ8LIYSwv2qHXLTWpUqpCcB3GNMW39dab1FKjat4fQqwAGOGSzbGtMWRNRdZCCFEVWxaKaq1XoBRtCs/N6XS7zUw3r7RhBBCXA3nbaYshBDiIlLQhRDCRUhBF0IIFyEFXQghXIRp3RaVUkeBvdf47fWAY3aM4wzknN2DnLN7uJ5zbqi1rnIdlWkF/XoopdZdrpeBq5Jzdg9yzu6hps5ZhlyEEMJFSEEXQggX4awFfarZAUwg5+we5JzdQ42cs1OOoQshhPhfznqFLoQQ4hJS0IUQwkU4dEFXSvVWSu1QSmUrpSZW8bpSSk2qeD1TKdXWjJz2ZMM5j6g410yl1AqlVJIZOe2punOudFx7pVSZUurO2sxXE2w5Z6VUmlIqQym1RSm1tLYz2psN/28HKaW+VkptrDhnp+7aqpR6Xyl1RCm1+TKv279+aa0d8gujVe8uIAHwAjYCLS45pi/wLcaOSTcAq83OXQvn3BmoW/H7Pu5wzpWO+xGj6+edZueuhT/nYGArEFvxONzs3LVwzv8HvFTx+zAgH/AyO/t1nHN3oC2w+TKv271+OfIVegcgW2u9W2tdDMwGBlxyzADgI21YBQQrpSJrO6gdVXvOWusVWusTFQ9XYewO5cxs+XMGeAT4DDhSm+FqiC3nPByYp7XeB6C1dvbztuWcNRCglFKAP0ZBt+9GpLVIa70M4xwux+71y5EL+uU2nr7aY5zJ1Z7PKIx/4Z1ZteeslIoGBgJTcA22/Dk3BeoqpZYopdKVUvfWWrqaYcs5vwkkYmxfuQl4TGtdXjvxTGH3+mXTBhcmsdvm1E7E5vNRSt2EUdC71miimmfLOb8GPK21LjMu3pyeLefsAbQDegC+wEql1CqtdVZNh6shtpzzLUAGcDPQCFiklFqutT5d0+FMYvf65cgF3R03p7bpfJRSbYBpQB+t9fFaylZTbDnnVGB2RTGvB/RVSpVqrb+onYh2Z+v/28e01meBs0qpZUAS4KwF3ZZzHgn8QxsDzNlKqT1Ac2BN7USsdXavX4485OKOm1NXe85KqVhgHnCPE1+tVVbtOWut47XWcVrrOGAu8LATF3Ow7f/tL4FuSikPpZQf0BHYVss57cmWc96H8RMJSqn6QDNgd62mrF12r18Oe4Wu3XBzahvP+U9AKPBWxRVrqXbiTnU2nrNLseWctdbblFILgUygHJimta5y+pszsPHP+QXgA6XUJozhiKe11k7bVlcp9QmQBtRTSuUCzwGeUHP1S5b+CyGEi3DkIRchhBBXQQq6EEK4CCnoQgjhIqSgCyGEi5CCLoQQLkIKuhBCuAgp6EII4SL+H1qT4iH0S6RjAAAAAElFTkSuQmCC\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(df_scores.threshold,df_scores.tpr,label='TPR')\n",
    "plt.plot(df_scores.threshold,df_scores.fpr,label='FPR')\n",
    "plt.legend()\n",
    "plt.show()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:47:36.749074100Z",
     "start_time": "2023-12-09T01:47:36.517094100Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 4.3.3 随机基线模型"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "outputs": [],
   "source": [
    "np.random.seed(1) # 固定随机种子以实现可重复性\n",
    "y_rand = np.random.uniform(0,1,size=len(y_test)) # 生成一个随机数字在0-1范围内的数字"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:47:50.643108200Z",
     "start_time": "2023-12-09T01:47:50.629077600Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [],
   "source": [
    "# 在不同阈值下计算TPR和FPR\n",
    "def tpr_fpr_dataframe(y_test,y_pred): # 定义接收实际值和预测值的函数\n",
    "    scores = []\n",
    "    thresholds = np.linspace(0,1,101)\n",
    "\n",
    "    for t in thresholds:\n",
    "        tp = ((y_pred>=t)&(y_test==1)).sum()\n",
    "        fp = ((y_pred>=t)&(y_test==0)).sum()\n",
    "        fn = ((y_pred<t)&(y_test==1)).sum()\n",
    "        tn = ((y_pred<t)&(y_test==0)).sum()\n",
    "        scores.append((t,tp,fp,fn,tn))\n",
    "\n",
    "    df_scores = pd.DataFrame(scores)\n",
    "    df_scores.columns = ['threshold','tp','fp','fn','tn']\n",
    "    df_scores['tpr'] = df_scores.tp /(df_scores.tp+df_scores.fn)\n",
    "    df_scores['fpr'] = df_scores.fp/(df_scores.fp+df_scores.tn)\n",
    "    return df_scores\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:48:32.558214500Z",
     "start_time": "2023-12-09T01:48:32.531218Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "outputs": [
    {
     "data": {
      "text/plain": "   threshold   tp    fp  fn  tn       tpr       fpr\n0       0.00  434  1327   0   0  1.000000  1.000000\n1       0.01  430  1314   4  13  0.990783  0.990203\n2       0.02  419  1298  15  29  0.965438  0.978146\n3       0.03  414  1282  20  45  0.953917  0.966089\n4       0.04  408  1270  26  57  0.940092  0.957046",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>threshold</th>\n      <th>tp</th>\n      <th>fp</th>\n      <th>fn</th>\n      <th>tn</th>\n      <th>tpr</th>\n      <th>fpr</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.00</td>\n      <td>434</td>\n      <td>1327</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.01</td>\n      <td>430</td>\n      <td>1314</td>\n      <td>4</td>\n      <td>13</td>\n      <td>0.990783</td>\n      <td>0.990203</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.02</td>\n      <td>419</td>\n      <td>1298</td>\n      <td>15</td>\n      <td>29</td>\n      <td>0.965438</td>\n      <td>0.978146</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.03</td>\n      <td>414</td>\n      <td>1282</td>\n      <td>20</td>\n      <td>45</td>\n      <td>0.953917</td>\n      <td>0.966089</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.04</td>\n      <td>408</td>\n      <td>1270</td>\n      <td>26</td>\n      <td>57</td>\n      <td>0.940092</td>\n      <td>0.957046</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 计算随机模型的TPR与FPR\n",
    "df_rand = tpr_fpr_dataframe(y_test,y_rand)\n",
    "df_rand.head()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:48:34.321087Z",
     "start_time": "2023-12-09T01:48:34.169092100Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "outputs": [
    {
     "data": {
      "text/plain": "<matplotlib.legend.Legend at 0x26714d7edf0>"
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3RUVdvG4d8z6SQkQAi9I70FCCV0KdJFEKULSBFRAUVf4dMXe0dFlCJFBEUQbCBF6V2UFuklJJSAtCAtEEjZ3x+DvhEDDGSSk5l5rrVYy5nZmbmPsG4OZ/bZW4wxKKWUcn02qwMopZRyDi10pZRyE1roSinlJrTQlVLKTWihK6WUm/C26oPz5s1rSpQoYdXHK6WUS9qyZcsZY0xYeq9ZVuglSpRg8+bNVn28Ukq5JBE5fLPX9JKLUkq5CS10pZRyE1roSinlJiy7hq6UUs6QlJREXFwciYmJVkdxKn9/f4oUKYKPj4/DP6OFrpRyaXFxceTMmZMSJUogIlbHcQpjDPHx8cTFxVGyZEmHf+62l1xE5DMROSUiO2/yuojIWBGJFpHtIlLjDnIrpVSGJCYmEhoa6jZlDiAihIaG3vG/Ohy5hv450OoWr7cGylz/NRCYcEcJlFIqg9ypzP9yN8d020I3xqwBzt5iSAdghrHbCOQSkYJ3nMRBZ08dY+P4gZw/eyazPkIppVySM2a5FAaOpnkcd/25fxGRgSKyWUQ2nz59+q4+LGbTImqdnMO1sRGsnz+V1JTUu3ofpZRyhvj4eMLDwwkPD6dAgQIULlz478ciQnh4OJUrV+ahhx7i8uXLAHh5ef39fPv27Tl37pxTsjij0NP7d0G6u2YYYyYZYyKMMRFhYeneuXpbEW0HcLjTAi5656H+1mfY9E5rzp85flfvpZRSGRUaGkpUVBRRUVEMGjSIp59++u/HgYGBREVFsXPnTnx9fZk4cSIAAQEBfz+fJ08exo0b55Qszij0OKBomsdFgExt2FLVGlBy5K/sqDCc8KtbMJ82hrgtmfmRSimVIQ0bNiQ6Ovpfz0dGRnLs2DGnfIYzpi3OB54UkdlAHeC8MeYPJ7zvLYmXD1W6jGLqnMq03PUcwZ+1wtbmXYjom9kfrZTKpl75cRe7j19w6ntWLBTMS+0rZeg9kpOTWbx4Ma1a/XN+SUpKCsuXL6dfv34Zev+/ODJtcRbwC1BOROJEpJ+IDBKRQdeHLAJigGhgMjDYKckc1KXD/fT1Hc02r8qwYBjMexKS3OsGA6WUa7py5Qrh4eFERERQrFixv4v7r+dDQ0M5e/YsLVq0cMrn3fYM3RjT7TavG+AJp6S5C0F+3gxrX5eHvvLm+4qrqbZtMpzYAQ/PgNzFrYqllLJARs+kne2va+U3e/78+fO0a9eOcePGMWTIkAx/nlus5dKmSgHql8lHz4MtON7qMzgbA5MaQ9QsSNVZMEqp7CkkJISxY8cyevRokpKSMvx+blHoIsJrHSrj622j2cJAfq43G/KUgh8GwWct4dhWqyMqpVS6qlevTrVq1Zg9e3aG30vsV0yyXkREhHH2BhcnLyQydPY2Nsac5aEahXiz1E58Vr4CCWegVj9o9hL4Bzv1M5VS1tqzZw8VKlSwOkamSO/YRGSLMSYivfFucYb+l/zB/szsX5chzcrwzbbjdN1UmnP9N0Kdx2DTVBhXB/YutDqmUkplCrcqdAAvm/BMi7KM616DHXHn6TxtF8ciX4b+yyAgN8zuDl/3gosnrI6qlFJO5bbL57apUpDcOXwZOGMzHcetp1aJPNiCx9Ay9Wva7P8CW8xqaPEK1OgNNrf7e00p5YHcuskiS4cyZ1AkRfPkYN/Ji+w5dYUX4lvSLultzoWUt89bn1gfdn4LqSlWx1VKqQxx2zP0v1QoGMy3j9f7+/HJC4n0/uw3IuKGMisyjlqHp8I3j0Let6HZKCjfDtxwKU6llPtz6zP09OQP9mfOoEhqlQjlofVFmFzlS0znz0Fs8HVP+OIBOL3P6phKKXXHPK7QAYL9ffj80Vq0rVqQNxYf4PVD5Uh9bB20fg+Ob4PxkbDhY6tjKqVcxF/L4f7169ChQ6xatYqQkBCqV69OhQoVeOWVVwD+8Xz58uV59tlnnZbD7S+53Iyftxcfd61OWJAfU9fFcuriVUY/1A+/yp1gwdOw5EW4HG+fu66XYJRSt5DeLf6HDh2iYcOGLFiwgISEBMLDw2nXrh3A389fuXKF6tWr07FjR+rXr5/hHB55hv4Xm014qX1FRrQuz4+/H6fvtE1c9AqBhz6HiEdh3Yf2L071C1OlVAYEBgZSs2ZNDh48+I/nAwICCA8Pz1bL57o0EWFQ49KEBfnx/LfbefjTjUzvW4t8bT8A/1yw7gPY8S2EloLQe6BsK6jUCbw8/n+dUtnP4hH2xfmcqUAVaP32LYf8tXoiQMmSJfn+++//8Xp8fDwbN27kv//9L2l3a/vzzz85cOAAjRo1ckpUbaXrHqxZhNAgXwbP3EqnCRt4rUNlIhu/iH/BanB4PcQfhCMb7VMcV70FDYdD1S7g5WN1dKWUxW62quLatWupXr06NpuNESNGUKlSJVatWsXatWupWrUq+/btY8SIERQoUMApObTQ02hSLh+zBtSl3/TN9P18Ezl8vWhwTxEGNBpJrRJ57Cs37lsEa96FeU/A8lchvDtU7wWhpa2Or5S6zZl0VvvrWvnNnt+/fz8NGjSgY8eOf5/hZ4RHX0NPT7WiuVj3/L1M61uLTjUKE3X0HF0+/YVPVhwgFYEK7WDgaujxDRSuCevHwsc17FMeL93dxtdKKc9UtmxZRo4cyTvvvOOU99NCT4e/jxf3lsvH6w9UYcWzTWhXtRCjl+yn97TfOHPpqn3WS5kW0G0WPL0LGj8P+3+G8XVgz49Wx1dKuZBBgwaxZs0aYmNjM/xebrV8bmYxxjB701Femr+LXAE+fNS1OpGlQ/856NQe+G4gnNhuXx+m7ft6fV2pLKDL5/6PnqE7QEToVrsYPwyuT5CfNz2mbGTs8gOkpKb5yzBfBei/HOoPg63T4asucPWidaGVUh5HC/0OVCwUzPynGtC+WiE+WLqf9h+v44dtx0hKub7NnbevfQXH+z+GmFUwrbUu06uUyjJa6HcoyM+bMV3CGdMlnKvJKQz7OopG765kxi+HSP6r2Gs8At2/hvgYmHQvHN1kaWal3J1Vl44z090ckxb6XRARHqhemKVPN2Zan1oUzZODUfN20e7jdWw6dNY+qEwL6Pez/Tr6tNawaQq44R86pazm7+9PfHy8W5W6MYb4+Hj8/f3v6Of0S1EnMMbw084TvLZgN8fPJ9K2SkEGNipFtaK54Mqf9i9LDyyBsq2hek+4pzn43NlvlFIqfUlJScTFxZGYmGh1FKfy9/enSJEi+Pj8c3LFrb4U1UJ3osvXkpmw6iCfrz/ExavJ1C6Rh2EtylCvZB77EgIbx9sX/PILhor3Q53HoUBlq2MrpVyIFnoWu5iYxNebjjJt/SFOXEjkzY6V6VKrGKQkQ+xq+/IBu76HpMtQshE0eBpKN7U6tlLKBWihWyThajKPz9zKmv2nGd6iLE82vQf5ayney2ft0xt/nQQXj0N4T2j1FvgHWxtaKZWt6Tx0iwT6eTO1dwSdqhfm/aX7eXn+rv99cZMjj/3MfOjv9oW+fv8KJtSD2DXWhlZKuSwt9Ezm42Xj/YerMaBhSab/cpg3Fu7557fx3r72vUwfXQJevjD9fljznn0hMKWUugO62mIWEBH+r00FklIMU9bFksPPm2dalP3noKK1YNBa+HEorHgdjm2DjhPAP8Sa0Eopl6OFnkVEhFHtKnLlWsr1ZQNSGdzkHgL90vwW+AZCp8lQOAKWvACTmkDnaVAo48tqKqXcn15yyUI2m/Bmpyp0ql6YcSsPEvnWct5evJeTF9LMnxWBuoOg9wJISoSpLWDjBL0pSSl1Ww4Vuoi0EpF9IhItIiPSeT1ERH4Ukd9FZJeI9HV+VPfgZRM+6BLOt4/Xo/49eZm05iBNR69i/u/H/zmweCQ8vh5KN4OfRsDMhyBuizWhlVIu4bbTFkXEC9gPtADigE1AN2PM7jRj/g8IMcY8LyJhwD6ggDHm2s3e1xOmLTri0JkEhs/9nS2H/6R7nWKMalcRfx+v/w0wBn79FFa+AVcvQNE6EPkElG8PNv0HllKeJqPTFmsD0caYmOsFPRvocMMYA+QU+yTrIOAskJyBzB6jRN5AZg+sy6DGpfnq1yM8OGFD+pdgntkNrd6BSydhziMwuQkcXGlZbqVU9uNIoRcGjqZ5HHf9ubQ+ASoAx4EdwFBjzL/m3YnIQBHZLCKb0+587el8vGyMaF2ez/pEcOhMAp3Gb+Dg6Uv/HOSX017sT22Fjp/C5T/hiwfgi45w4Xj6b6yU8iiOFLqk89yN12laAlFAISAc+ERE/nXLozFmkjEmwhgTERYWdsdh3V3T8vn5+rFIrian0HnCBrYe+fPfg2xeUK0rPLUZWr4FR3+DTxtB7NqsD6yUylYcKfQ4oGiax0Wwn4mn1Rf4zthFA7FAeedE9CyVC4fw7eP1CAnwodukjXz165H0lwX19oPIwTBgBQTkhhkdYMPHOhtGKQ/mSKFvAsqISEkR8QW6AvNvGHMEaAYgIvmBckCMM4N6kuKhgXz7eD1ql8zD/32/g6Gzo7h09SZfSYSVs5d6+baw5EWY20e3vlPKQ9220I0xycCTwM/AHmCOMWaXiAwSkUHXh70G1BORHcBy4HljzJnMCu0JQoP8mN63Ns+1LMeC7cdp/dEaZv56mMSklH8P9ssJD8+AFq/BnvkwuRmc3p/1oZVSltLVFl3ArzHxvLFoD9vjzpMn0JfekSUY1KQUft5e/x4cuwbm9oXkRGj9LoR3t8+UUUq5BV0+1w0YY/gt9iyT18aybM9JqhQOYVz3GhQLzfHvweePwXcD4PB6KNcG2n8EQfmyPrRSyul0+Vw3ICLUKRXKlN4RTOpVk8PxCbQdu5ZFO/749+CQwvalA+57A6KXw/i6EL0s60MrpbKUFroLuq9SARYOaUipfEEMnrmVgTM2c/Ts5X8Ostmg3pPw2BoIKgBfdobVuiyvUu5MC91FFc2Tg7mPRfJcy3KsPXCGFh+uZuzyA1xMTPrnwHzlof8yqPIQrHwdZneDS3pTl1LuSK+hu4Fj567wxsLdLNpxgpx+3nStXZQ+9UtSOFfA/wYZA79Nhp9Hgk8g3DsSavUHL5+bv7FSKtvRL0U9xPa4c0xeG8uiHX8gwH9alaN/g1LYbGlmuZzeB4ufh5iVEFYBGj8HFTqAly6Nr5Qr0EL3MHF/Xua1Bbv5eddJmpXPx+iHqpE70Pd/A4yBfYtgyX/h7EEILgK1B9jP2P2CrAuulLotLXQPZIxh+oZDvLFoD2FBfkzuHUGlQjdsZ5eaCgd+hl/GwaG1EFIM2n0AZVpYE1opdVs6bdEDiQh96pfk28frYYAun25kffQNN+/abFCuNfRZAH1/Ap8AmNkZvumnX5wq5YK00N1c1SK5+G5wPQrl8qfPtN/+vTPSX4pH2jepbjLSvnzAJxGw7Utd7EspF6KF7gEKhgQw97F6VC+amyGztjF1XWz6A739oMkIGLQOwsrDvCdgxv32re+02JXK9rTQPURIDh9m9KtNq0oFeG3Bbt5atIfU1JuUdFg56LsY2n4Ax6NgSlOY2MC+Fd7VS+n/jFLKclroHsTfx4txPWrQq25xPl0TwzNzorh87SbL8tpsUKsfPL0T2n1on6+++D8wqQmc2puluZVSjtFZLh7IGMO4ldGMXrKfXDl86FmnOI/UK06+nP63/sGY1fBtP7h2GR4YB5U6Zk1gpdTfdNqiSteWw2eZtCaGJbtP4mOz0ad+CYY0K0OQ3y1uMrpwHOb0hrjfoPEI+zV3XZ5XqSyjha5uKfZMAuNWRvPNljjyB/vxQtuKtK9aELlZUSdfgwXDIGomNHgamr2kpa5UFtF56OqWSuYNZPRD1fhucD3CcvoxZNY2hs/9naSUm6zM6O0L938CEY/Cug/tW9/pLBilLKeFrv5Wo1hu5j3RgGHNy/Dd1mP0n76ZhJvtZWqz2WfB1B4Iv3wCc3pB/MGsDayU+gctdPUPXjZhWPOyvN2pCmsPnKbb5I2cvng1/cEi9m3umo2yb6QxrjYsHK53mSplES10la6utYsxqVcE+09epO3YtWyMiU9/oAg0HA5DtkGN3rB5GkxuqmfrSllAC13dVPOK+fl+cH2C/LzpPnkjHy8/QMrNbkbKWcC+sFf/ZZCUAJ+1ghM7sjawUh5OC13dUoWCwcx/qgH3VyvE+0v3c9+Hq5n12xESk1LS/4HCNewLfXn5wLS2ELs2awMr5cG00NVtBfl582GXcMZ1r0GArxcjv9tB/bdXMGVtTPozYcLKwqM/Q1A+mN7efl098ULWB1fKw+g8dHVHjDFsjDnL+FXRrD1whjL5gni1Q2UiS4f+e/DVS7Didfh1IuQsCI2GQ8kmEFpa560rdZf0xiKVKZbtPsnLP+4i7s8rlM0fRKm8QZQKC6TBPXmJLB36vxuT4rbAj0Ph5PVr6kH5oeID0PQF8A+5+Qcopf5FC11lmsSkFD7fcIjNh84ScyaBI/GXSU41VCwYTP+GJWlXtRC+3jb7jUfx0XBoHcSugd0/QGA+aPMeVLzf6sNQymVooassk5iUwryoY0xeG0v0qUsUCPand70SdK9TjJAAn/8NPLYVfhxinwlTrg20fBPylLQuuFIuQgtdZbnUVMPq/aeZsi6G9dHx5PD1on3VQjSvmJ8G9+QlwNcLUpLs+5mufhdSk6H+EGjwDPjmsDq+UtmWFrqy1K7j55m6LpYlu05y6Woyft42OoQXYlT7SvaVHS8ch6WjYMdcCC1j3+M0ZwGrYyuVLWmhq2zhWnIqv8We5addf/DVr0coERrIJ91rULFQsH1AzCqY1R1CikCfhRAUZmlepbKjDK+2KCKtRGSfiESLyIibjGkiIlEisktEVmcksHJPvt42GpTJy+sPVGHWgLokXEvmgfHrGbcymkNnEqBUE+gxF84dgRkd4PJZqyMr5VJue4YuIl7AfqAFEAdsAroZY3anGZML2AC0MsYcEZF8xphTt3pfPUNX8Zeu8tw321mx1/5HpVRYIPVL56WBbQctooaQmrsk3p0m2u8+VUoBGbzkIiKRwMvGmJbXH48EMMa8lWbMYKCQMeZFR0Npoau/HIm/zIq9J1m+9xTbjpzj0tVk6tl28qHPePLKBaIKdUXufYFShfORK4ev1XGVslRGC70z9jPv/tcf9wLqGGOeTDNmDOADVAJyAh8ZY2ak814DgYEAxYoVq3n48OG7OyLltowxnLl0jZjTl9h58ChFt77DfVcWcTQ1jGeTBrE/oCrVi+Xm1Q6VKJJbZ8Moz5PRa+jp3aN9498C3kBNoC3QEviviJT91w8ZM8kYE2GMiQgL0y+81L+JCGE5/ahTKpR+LcK57/lZnO86n9DgAGb7vc7YPN/we+wJ2ny0liW7TlgdV6ls5Ra7Af8tDiia5nER4Hg6Y84YYxKABBFZA1TDfu1dqQwJKd8YSv4CS0fRcPNUfgmN4onU4Qz8YgtdaxUlsnQoJfMGUjosiMBbbXCtlJtz5JKLN/ZibgYcw/6laHdjzK40YyoAn2A/O/cFfgO6GmN23ux99Rq6uivRy+Db/hhgepFXeW1X3r/XaM/p583MAXWoWiSXtRmVykQZuuRijEkGngR+BvYAc4wxu0RkkIgMuj5mD/ATsB17mU+5VZkrddfuaQ4DViCB+ehz8Gn2dviDpcMaMrFnTYIDfBgwYzMnzidanVIpS+iNRco1JZ6Hb/pB9FKo0B7ajWHPBV86T9hAqbAg5jwWaV9eQCk3k+Ebi5TKdvxDoPvX0OJV2P8zjK9LhXOr+ahLODuPn2f43CiS09t8Qyk3poWuXJfNC+oPhYGr7Gu/fN2T5ms6My38AMt2HKXb5I38cf6K1SmVyjJa6Mr15a8E/VfA/R+DSaHJnpfYnutZChxfTpuP1rJy3y1vWlbKbeg1dOVejLEv8rV0FJzYzlKfpjxzsTtVShdlQMNSNC4bhs2m298p16XX0JXnEIHS90L/5dDoOZonr2ZDyIvkP7mGvp9vouWYNWw98qfVKZXKFFroyj15+0LTF5F+S8kZnIsPk99gbekv8L8Wz8MTf2HK2his+tepUplFC125tyI14bE10OT/KHpiOfPNMMbmW8CnCzcwYMZmTl3UOevKfeg1dOU5Tu+DZS9j9i0mVbz4MSWSr6QtLVu0ondkcby99PxGZX+6Y5FSacUfhF8/JXXbF9iSLvNrankWBz1I5+4DqazLBqhsTgtdqfRcOYfZOoPE9RMIuHycFak1ONPsAx5qFI6IzoRR2ZPOclEqPQG5kPpDCBi+g4Smr9PQtoOGKzoxZso0LiQmWZ1OqTumha6UlzeBjZ7Ca8AyAnLkZEjcMywZ3ZtdB3UDFuVatNCVus5WOJxcwzYQX6EHHZMXU2hGfTZ8/R4mJdnqaEo5RAtdqbT8gsjXdRwJjyzjlH8J6u15nc1vtWD+Lzu5lqyLfansTQtdqXQEl6pJmf+sYWuVUVRL3k71xQ/Q7+0pbDh4xupoSt2UFrpSN2HzslHjweH49P+ZsCBvpiS/wLYvRhBzJM7qaEqlSwtdqduQIhH4P7GOlHta8ARzyfdZLRJ/egmu6JowKnvRQlfKEYF5ydHzK3Z3WMSa1Kr4bvwI80lt2PW9fYVHpbIBLXSl7kDF6vVJfGAq7a++Qey1EJjbB2Z1td99qpTFtNCVukOdahShV8f2tL3yMmNsvUmJWQ0f14SvukLsGj1jV5bRQlfqLnStXYxvBzdifo5ORCa8z2/F+mHifoPp7eHrnpCkW9+prKeFrtRdqlgomPlPNSCyWkUe3t+Ufrk/J6Hhi7B3ob3YE+Ktjqg8jBa6UhkQ5OfNmC7hvN2pCusPJ9BkY3U21xmDObEDpraA0/utjqg8iBa6UhkkInStXYwfnqhPrgAfOq8OY7DXSyRejMdMiISFw+HiSatjKg+gha6Uk1QoGMxPwxoxsWdNTucKp+HFN/kqqQkpmz4jaUw1Li17F3RdGJWJdD10pTLJ9rhzLN55gr07t9L1/FRaem3mZHAV8vSchk++MlbHUy5KN7hQymLRJy+yfO54up4eg58kEx/xNIVbDAG/IKujKRejG1woZbF78ufksSefZ8f9P7HZVpXCm98h5cMqsGY0JF6wOp5yE1roSmWhBjWrUWTwfHqY14lKLQ0rXoOJ9eHkbqujKTegha5UFiuRN5AunR7kwQtPM7PSJEi+Zp/iuO8nq6MpF6eFrpQF7q9WiG61i/LCliDWNZ0DoffY14RZ+z6kplgdT7kohwpdRFqJyD4RiRaREbcYV0tEUkSks/MiKuWeRrWrRPkCOek1N45XwkZzrfwDsPxV+LwtnI21Op5yQbctdBHxAsYBrYGKQDcRqXiTce8APzs7pFLuKMDXizmDIulTrwTTN52izr5ubKnxNpzcBRPqw7YvrY6oXIwjZ+i1gWhjTIwx5howG+iQzringG+BU07Mp5RbC/b34aX2lVg4pCGl8uXkwQ3FmFbtK0yRmjDvCVj2MqTqXqbKMY4UemHgaJrHcdef+5uIFAY6AhNv9UYiMlBENovI5tOnT99pVqXcVoWCwcweWJdO1QvzypoLvBT8Kqk1+8K6D+H7x+xfnCp1G44UuqTz3I13I40BnjfG3PLbHGPMJGNMhDEmIiwszNGMSnkEHy8b7z9cjUGNSzPj1+MMjO9BYqMXYcccmPkgXL1odUSVzTlS6HFA0TSPiwDHbxgTAcwWkUNAZ2C8iDzglIRKeRARYUTr8rxyfyVW7T9N8001OdzoQzi0XpfkVbflSKFvAsqISEkR8QW6AvPTDjDGlDTGlDDGlAC+AQYbY35welqlPETveiWYMygSY6D58gLMK/8uqSf3wLTWcP6Y1fFUNnXbQjfGJANPYp+9sgeYY4zZJSKDRGRQZgdUylPVKJabRUMa0qx8foZuK0i3y8+RcOYIl8Y1ImXPAqvjqWxIF+dSygUcO3eFFXtOsjdqAz3/eJMKtqOklG+PV9vRkLOA1fFUFtLVFpVyI1NX7yN+yfsM8/kOHx9fpHZ/iHwKgnSigSfQ1RaVciP9Gpej/MMv0zrpXZabmpgNH8OYKrD4eTi11+p4ykJ6hq6Ui9pw8AzPzd2O7/kY3glbQq1Ly5HUZChaB2r2gcqdwdvX6pjKyfSSi1Ju6sq1FMaviubT1TGE2S7w36K/0+zyT/icOwghRaHBMKjeC7z9rI6qnEQLXSk3d+hMAuNWRjMv6jhJqSkML3GEAeYb/E5sgeDC0PodqNDe6pjKCbTQlfIQpy4kMuOXw0xbH0tSaipvVo2n05kJ2E7tgvLtoM1oCC5odUyVAVroSnmYkxcSeWPhHub/fpxSeXz5ofo2gn99H7z8oP2HUPlBqyOqu6SzXJTyMPmD/RnbrTpf9a/DyUspdN8byZX+ayCsLHzzKMwfAtcuWx1TOZkWulJurN49efm4e3V2Hb/AM8sukdp7ETR4GrZOh8lN4dwRqyMqJ9JCV8rNNS2fnxfaVGDxzhN8sCIWmr8MPb+DC8dhWlv485DFCZWzaKEr5QH6NShJl4iifLIymhd/2EFi8SbQex5cvWAv9fiDVkdUTqCFrpQHEBFe71iZxxqV4suNR+g0fgOxvmWhzwJIugzT2sCeBWDRJAnlHFroSnkIHy8bI9tU4LM+ERw/f4V2Y9cy5UAgSY/8CH454eseMPU+OLzB6qjqLmmhK+VhmpbPz6IhDYkokYfXF+6h3eyzbGy9ENp/ZP+SdFprmPGAFrsL0kJXygMVyhXA531rMalXTS5dTabrlM0MPVCNk31/gRavwsmd9mKf1hb2L9GNql2E3liklIe7ci2FCauimbgmBh+bMLR5GR6uFkqu3V/BhrFw8Q8ILQN1H4fw7uATYHVkj6Z3iiqlbuvQmQRe+XEXK/edxssm1Cyem5bl8tAt5xZybP4U/oiCoPxQbwhE9AXfQKsjeyQtdKWUQ4wx/B53nmW7T7Jsz0n2nrhI3sM7nk4AAA4QSURBVCA/RrYqR6fQQ8ia9yB2NeQIhVZvQ9WHrY7scbTQlVJ3ZXvcOf47bxe/Hz1HRPHc9IosTrPAwwSteRmO/grhPaD1u+AXZHVUj6GFrpS6a6mphrlbjjJ6yX5OX7yKl02oUzyYUTl/pNz+T5HQe6DrTAgrZ3VUj6CFrpTKsJRUQ9TRc6zYe5LFO08QczqBNkEHGG0bS4C3IL3nQ/5KVsd0e1roSimnSk01rD5wmslrYvgjZiffBLxFHr9U5JF5ULCq1fHcmi6fq5RyKptNuLdcPr4aUJchD7ei05UXOHvNCzO9PRxYqksIWEQLXSmVIR2rF2FQx+Z0uPwip1KCYGZnmHE/HNtidTSPo4WulMqwbrWL0bdtYxpcfJPJQYNI/mOnfb317wdBQrzV8TyGFrpSyin6NSjJu11qMvFKc2qcf5fV+XphdsyFcbXg96/1MkwW0EJXSjlNx+pFWPFsEzpFVqTv0dYMDPiAa8HF4fuB8FlLiF6mxZ6JtNCVUk4VEuDDy/dXYmb/umy8VIAm8SM50ehtOH8MvnwQpjSDmNVWx3RLWuhKqUwRWTqUOYMiSREb960pxS9tl9qX6E04bf/S9PvH4fJZq2O6FS10pVSmqVAwmG8fr0e+YH96fL6Nj/6sT8rjv0LD4bBjDnxSC/Yusjqm23Co0EWklYjsE5FoERmRzus9RGT79V8bRKSa86MqpVxRkdw5mPdEfTqEF+bDZfvpNWM7p2r9BwauhpDC9p2Stn5hdUy3cNtCFxEvYBzQGqgIdBORijcMiwUaG2OqAq8Bk5wdVCnlugL9vPng4Wq8+2BVthz+k2bvr2bawUCSH1kIpe6F+U/C+rFWx3R5jpyh1waijTExxphrwGygQ9oBxpgNxpg/rz/cCBRxbkyllKsTER6uVZTFQxsSXiwXr/y4m3afbmN7w4lQqSMs/S980w8OroCUZKvjuiRHCr0wcDTN47jrz91MP2Bxei+IyEAR2Swim0+fPu14SqWU2ygVFsSMR2szoUcNzl9J4uGpW1hW4U2oPxT2/wxfdIT3y8GqdyA1xeq4LsWRQpd0nkt3IqmI3Iu90J9P73VjzCRjTIQxJiIsLMzxlEoptyIitK5SkB+fakC5/DkZOHMbs0P6w3PR0OVLKFobVr1pn+aod5o6zJFCjwOKpnlcBDh+4yARqQpMAToYY/R3QCl1W3mD/PhqQF0alQ1jxHc7eGNJDJdLt4Zus6D9WDi8ASY1hti1ekOSAxwp9E1AGREpKSK+QFdgftoBIlIM+A7oZYzZ7/yYSil3FejnzeRHIuhRpxiT18bS/P3VLNrxB6bGI/DoT/ZB09vBR1Vh6UtwJtrawNmYQ+uhi0gbYAzgBXxmjHlDRAYBGGMmisgU4EHg8PUfSb7Zer1/0fXQlVI32nzoLP+dt4s9f1ygUqFg7qtYgBalc1Dh/Gpk53cQsxK8/KDHHCjRwOq4ltANLpRSLiM5JZXZm47y7dY4oo6ewxgoGOJP0/L5aFM8lcgN/bGdOwrdZ0OpJlbHzXJa6Eopl3T64lVW7jvF8j0nWXvgDJevpVAxOJFvcrxDjkuHoctMKNPc6phZSgtdKeXyriansD76DK/8uJvLf55kce73Cb0cjdQfCk1Ggref1RGzhG5Bp5RyeX7eXjQtn58fn2pA7UplaRL/H1bnuA/WfWjfTOPEDqsjWk4LXSnlUoL9ffike3VGPFCbJy89ysDk50g4exwzqQksfxWSEq2OaBktdKWUyxERetYtzorhjclRuS31L77FYhrC2vcxE+rBoXVWR7SEFrpSymXlC/ZnTNfqTBzYgo9yPkOPayM5feEyZnp72PCJx92MpIWulHJ5dUuFsmBIA5q26UK7pHdYmloLlrxgX8Ux+ZrV8bKMt9UBlFLKGXy8bPRrUJI2VQrQ7dMgYi9/yWPbvoTT+6HNe1Ao3OqImU7P0JVSbqVgSABfDohkul9PRsgwUs4csK8H800/OBtrdbxMpYWulHI7RXLnYOaAuiz3bkjTa2PYW3YgZu9CGFcHdnxjdbxMo4WulHJLJfMGMmtAXULy5KXV9iYMyj2Fy2HV4Nt+sOJ1SE21OqLTaaErpdzWPfmC+H5wfd7qVIXf4n2pceQptoa2hzXvwdxH4FqC1RGdSgtdKeXWvGxCt9rFWDG8CQ/WKsmDx7vyga0PZs9C+KwlnI+zOqLTaKErpTxC7kBf3uhYhXlPNGB16MP0vfYsV08dxEy6F+LcY10pLXSllEepWiQX3z1ejxJ1H6DdlZeJv+aFmdYGfpvs8jciaaErpTyOl014qX1FOrVsTouLLxHlXRUWPQtf94TLZ62Od9f0xiKllEcSER5vUpp8Of3o8UMI3VMXMGLvbFKO1MOv9etQqRPYXOuc17XSKqWUkz1YswgbRjYnd/Nn6Of1JjGXfOxTGyc3gYMrrY53R3SDC6WUuu5qcgqPTd9EaOw83giZh3/CMWjwNDQdlW3O1nWDC6WUcoCftxdje9Tk9zytaHj5PS5W6mXfQOP7x1xikS8tdKWUSiPY34epvSNIFh8eOPwg5yJHwo458GUnSIi3Ot4taaErpdQNiocGMrFnTU5cuErttdVYXOYVzJGNML4u7Ftsdbyb0kJXSql01CkVyvLhTWhZqQCP7yhDH593ueofCrO6wrwnIPGC1RH/RQtdKaVuokCIPx93q86sAXXZlVKUNlde5UrdYRD1FUyoD7FrrY74D1roSil1G5GlQ/m0VwRHz6fQ90grknr/BF4+ML0d/DQSrl60OiKgha6UUg6pWTw373SuwsaYs4zaGkDygNVQeyBsHA9jqsDq9yDxvKUZdR66UkrdgXd/2sv4VQfx8RKK5clB05xxPC7fkefYcvALgXYfQJXOmfb5t5qHrrf+K6XUHXj2vnJULBTMruMXiD2dwPyjBZh8oR+Ple3C8OQp+H7bDy4cg3pDQCRLs+kZulJKZUBiUgoTVx9kwqqDBNiSmFfoS4qf+BlqDYDW74DNy6mfp3eKKqVUJvH38WJY87Ise6YxlYvlp8mhXizP3QU2TYaZD2Xp6o1a6Eop5QRF8+RgxqO1ebZlBQac6MBovydIjV0LkxrD8agsyeBQoYtIKxHZJyLRIjIinddFRMZef327iNRwflSllMrebDbhiXvvYdaAusw1TXn42igSEq9hpt4HGydCakrmfv7tBoiIFzAOaA1UBLqJSMUbhrUGylz/NRCY4OScSinlMuqUCmXRkIbkKFWHhudeZrdfOPz0PClTWsDJXZn2uY6codcGoo0xMcaYa8BsoMMNYzoAM4zdRiCXiBR0clallHIZoUF+fN6nFv1a1uL+c0MZem0w547tJ3lCQ36Z+VqmfKYj0xYLA0fTPI4D6jgwpjDwR9pBIjIQ+xk8xYoVu9OsSinlUv66BNOuakF2HqvB9390otru95A8JTPl8xwp9PQmUt4419GRMRhjJgGTwD5t0YHPVkopl1c8NJDioYFQtSC0nJNpn+PIJZc4oGiax0WA43cxRimlVCZypNA3AWVEpKSI+AJdgfk3jJkPPHJ9tktd4Lwx5o8b30gppVTmue0lF2NMsog8CfwMeAGfGWN2icig669PBBYBbYBo4DLQN/MiK6WUSo9Da7kYYxZhL+20z01M898GeMK50ZRSSt0JvVNUKaXchBa6Ukq5CS10pZRyE1roSinlJixbD11ETgOH7/LH8wJnnBjHFegxewY9Zs+QkWMubowJS+8Fywo9I0Rk880WeHdXesyeQY/ZM2TWMeslF6WUchNa6Eop5SZctdAnWR3AAnrMnkGP2TNkyjG75DV0pZRS/+aqZ+hKKaVuoIWulFJuIlsXuiduTu3AMfe4fqzbRWSDiFSzIqcz3e6Y04yrJSIpItI5K/NlBkeOWUSaiEiUiOwSkdVZndHZHPizHSIiP4rI79eP2aVXbRWRz0TklIjsvMnrzu8vY0y2/IV9qd6DQCnAF/gdqHjDmDbAYuw7JtUFfrU6dxYccz0g9/X/bu0Jx5xm3Arsq352tjp3Fvw+5wJ2A8WuP85nde4sOOb/A965/t9hwFnA1+rsGTjmRkANYOdNXnd6f2XnM3RP3Jz6tsdsjNlgjPnz+sON2HeHcmWO/D4DPAV8C5zKynCZxJFj7g58Z4w5AmCMcfXjduSYDZBTRAQIwl7oyVkb03mMMWuwH8PNOL2/snOh32zj6Tsd40ru9Hj6Yf8b3pXd9phFpDDQEZiIe3Dk97kskFtEVonIFhF5JMvSZQ5HjvkToAL27St3AEONMalZE88STu8vhza4sIjTNqd2IQ4fj4jci73QG2RqosznyDGPAZ43xqTYT95cniPH7A3UBJoBAcAvIrLRGLM/s8NlEkeOuSUQBTQFSgNLRWStMeZCZoeziNP7KzsXuiduTu3Q8YhIVWAK0NoYE59F2TKLI8ccAcy+XuZ5gTYikmyM+SFrIjqdo3+2zxhjEoAEEVkDVANctdAdOea+wNvGfoE5WkRigfLAb1kTMcs5vb+y8yUXT9yc+rbHLCLFgO+AXi58tpbWbY/ZGFPSGFPCGFMC+AYY7MJlDo792Z4HNBQRbxHJAdQB9mRxTmdy5JiPYP8XCSKSHygHxGRpyqzl9P7KtmfoxgM3p3bwmEcBocD462esycaFV6pz8JjdiiPHbIzZIyI/AduBVGCKMSbd6W+uwMHf59eAz0VkB/bLEc8bY1x2WV0RmQU0AfKKSBzwEuADmddfeuu/Ukq5iex8yUUppdQd0EJXSik3oYWulFJuQgtdKaXchBa6Ukq5CS10pZRyE1roSinlJv4fZ6VRQkD4uWQAAAAASUVORK5CYII=\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(df_rand.threshold,df_rand.tpr,label='TPR')\n",
    "plt.plot(df_rand.threshold,df_rand.fpr,label='FPR')\n",
    "plt.legend()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:48:40.730110400Z",
     "start_time": "2023-12-09T01:48:40.568111200Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 理想模型"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "outputs": [
    {
     "data": {
      "text/plain": "     threshold   tp    fp   fn    tn       tpr       fpr\n0         0.00  434  1327    0     0  1.000000  1.000000\n1         0.01  434  1309    0    18  1.000000  0.986436\n2         0.02  434  1291    0    36  1.000000  0.972871\n3         0.03  434  1274    0    53  1.000000  0.960060\n4         0.04  434  1256    0    71  1.000000  0.946496\n..         ...  ...   ...  ...   ...       ...       ...\n96        0.96   71     0  363  1327  0.163594  0.000000\n97        0.97   53     0  381  1327  0.122120  0.000000\n98        0.98   36     0  398  1327  0.082949  0.000000\n99        0.99   18     0  416  1327  0.041475  0.000000\n100       1.00    1     0  433  1327  0.002304  0.000000\n\n[101 rows x 7 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>threshold</th>\n      <th>tp</th>\n      <th>fp</th>\n      <th>fn</th>\n      <th>tn</th>\n      <th>tpr</th>\n      <th>fpr</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.00</td>\n      <td>434</td>\n      <td>1327</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.01</td>\n      <td>434</td>\n      <td>1309</td>\n      <td>0</td>\n      <td>18</td>\n      <td>1.000000</td>\n      <td>0.986436</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.02</td>\n      <td>434</td>\n      <td>1291</td>\n      <td>0</td>\n      <td>36</td>\n      <td>1.000000</td>\n      <td>0.972871</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.03</td>\n      <td>434</td>\n      <td>1274</td>\n      <td>0</td>\n      <td>53</td>\n      <td>1.000000</td>\n      <td>0.960060</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.04</td>\n      <td>434</td>\n      <td>1256</td>\n      <td>0</td>\n      <td>71</td>\n      <td>1.000000</td>\n      <td>0.946496</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>96</th>\n      <td>0.96</td>\n      <td>71</td>\n      <td>0</td>\n      <td>363</td>\n      <td>1327</td>\n      <td>0.163594</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>97</th>\n      <td>0.97</td>\n      <td>53</td>\n      <td>0</td>\n      <td>381</td>\n      <td>1327</td>\n      <td>0.122120</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>98</th>\n      <td>0.98</td>\n      <td>36</td>\n      <td>0</td>\n      <td>398</td>\n      <td>1327</td>\n      <td>0.082949</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>99</th>\n      <td>0.99</td>\n      <td>18</td>\n      <td>0</td>\n      <td>416</td>\n      <td>1327</td>\n      <td>0.041475</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>100</th>\n      <td>1.00</td>\n      <td>1</td>\n      <td>0</td>\n      <td>433</td>\n      <td>1327</td>\n      <td>0.002304</td>\n      <td>0.000000</td>\n    </tr>\n  </tbody>\n</table>\n<p>101 rows × 7 columns</p>\n</div>"
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_reg = (y_test==0).sum() # 计算数据集中正类和负类的数量\n",
    "num_pos = (y_test==1).sum()\n",
    "y_ideal = np.repeat([0,1],[num_reg,num_pos])# 生成一个数组,该数组首先生成num_reg个0,然后生成num_pos个1\n",
    "y_pred_ideal = np.linspace(0,1,num_reg+num_pos)\n",
    "\n",
    "df_idea = tpr_fpr_dataframe(y_ideal,y_pred_ideal)\n",
    "df_idea"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:56:15.777020700Z",
     "start_time": "2023-12-09T01:56:15.730987400Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "outputs": [
    {
     "data": {
      "text/plain": "<matplotlib.legend.Legend at 0x26717a5aca0>"
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXwUdZ7/8denOwkBEgiEcKUTDgW5jxyg7qggIARQDAEJoCIgLKPOsbP+VmacGXWdyx1ndtZR1wFEFJWAHHIIKOCBFwMJNyIQASEBBMIpEMjx/f1R6AYMpEO6u7qrP8/HI48H6S663iXyplL1re9XjDEopZQKfS67AyillPINLXSllHIILXSllHIILXSllHIILXSllHKICLt23KhRI9OyZUu7dq+UUiEpLy/vqDEmobL3bCv0li1bkpuba9fulVIqJInI11d6Ty+5KKWUQ2ihK6WUQ2ihK6WUQ9h2DV0ppXyhpKSEgoICiouL7Y7iU9HR0Xg8HiIjI73+PVroSqmQVlBQQGxsLC1btkRE7I7jE8YYioqKKCgooFWrVl7/viovuYjIdBE5LCJbr/C+iMhzIpIvIptFJKUauZVSqkaKi4uJj493TJkDiAjx8fHV/qnDm2voM4ABV3k/A2hz8Wsi8L/VSqCUUjXkpDL/zrUcU5WXXIwxq0Wk5VU2GQK8Zqx5eNeISJyINDPGHKx2Gi/k7/maU+/9gU+SH6LUHe2PXSilqiHS7SK7RzIJsbXsjhL2fHENPRHYX+H7gouv/aDQRWQi1lk8ycnJ17SzU9tX0u3AbMoL1zOx5Bccp941fY5SyjeMgbV7j/HauB6OPFOuSlFREX369AHg0KFDuN1uEhKsBzk3bdpE165dKS0tpX379rz66qvUqVMHt9tN586dKS0tpVWrVsycOZO4uLgaZ/FFoVf2J1jpqhnGmCnAFIC0tLRrWlkjZeB4aBFP2vyJrE94FkbPhYbe3zRQSvnWjE/38OTiL1i65RCDujSzO07AxcfHs3HjRgCefPJJYmJiePTRRwGIiYn5/r3Ro0fz0ksv8Ytf/ILatWt///qYMWN44YUXePzxx2ucxRfj0AuApArfe4ADPvjcK+t4N9y/EM4Wwcv9oDDPr7tTSl3ZvTe2oEOzejy95Au+PV9qd5ygdcstt5Cfn/+D12+66SYKCwt9sg9fnKEvAh4RkRygJ3DSX9fPL9HiJhj3HryRBTMGw/AZ0La/33erlLpUhNvF7zI7MfTFz/iflTt5fFAH27I8tXgbXxw45dPP7NC8Hk/c2bFGn1FaWsqyZcsYMODS8SVlZWWsWrWK8ePH1+jzv+PNsMVZwOfADSJSICLjRWSSiEy6uMlSYDeQD0wFHvJJMm8ktIXxK6FRW5iVDbmvBGzXSqn/k5LcgOz0JKZ/upcdh07bHSdonDt3jm7dupGWlkZycvL3xf3d6/Hx8Rw7dox+/fr5ZH/ejHIZWcX7BnjYJ2muRWwTeOAdmDsWlvwcjn0FfZ8Cl9u2SEqFo8cGtGPJ5oNMWb2bv9zT1ZYMNT2T9rWK18ore/3kyZMMHjyYF154gZ/+9Kc13p8z5nKpFQPZsyB9Anz2d8gZBcW+/bFLKXV1DepGMahzM5ZtPcgZvZbulfr16/Pcc8/x7LPPUlJSUuPPc0ahA7gjYNCzMOgvsGsFTO8PJ/bZnUqpsDIszcPZC2Us33rI7igho3v37nTt2pWcnJwaf5bz5nJJfxDi28Cc+2BaXxg1B5p3szuVUmEhrUUDWsTXYW5eAVmpHrvjBNyTTz55yffffvttpdtd/vrixYt9sn/nnKFX1Po2awSMOwpeGQi7VtqdSKmwICIM7e7h891FFBw/a3ecsOPMQgdo3A7Gr4D41vDmPbB+pt2JlAoLQ1MSAViw3jdjq5X3nFvoAPWawdhl1hn7okfggz9azykrpfwmqWEderZqyPwNhRj9+xZQzi50gFqx1nX0bvfCR3+ChQ9DWc3vJiulriwr1cOeo2dYv++43VHCivMLHcAdCUOeh9smw8Y34PWhcPaY3amUcqyBnZtRO9LNW7kFdkcJK+FR6AAi0PuXcPdLsG8NTOsDR3banUopR4qpFcHgLs1YtOmAzu8SQOFT6N/pNhLGLIHzp61hjV99YHcipRwpu0cSZy+UsWSTf+fqCwZut5tu3bp9/7V3714+/PBD6tevT/fu3Wnfvj1PPfUUwCWvt2vX7vuZGX0h/AodILknTHgf6nvgjWGwcZbdiZRynJTkBrRpHEPOuv1VbxzivnuU/7uvli1bAtYMixs2bCA3N5fXX3+dvLy8S17fsGEDS5Ys4dNPP/VJjvAsdIC4ZBi3DFrcDG9PgtV/1hEwSvmQiJDdI5mN+0/w5aHwnoqjbt26pKam8tVXX13yeu3atenWrVtQTZ8buqLrw+h51pDG938Hx/bC4L9ChC6lpZQvZHZP5JllX5Kzdj9P3hWAibOWTYZDW3z7mU07Q8afrrrJd7MnArRq1YoFCxZc8n5RURFr1qzhN7/5DUeOHPn+9ePHj7Nr1y5uvfVWn0QN70IHiIiCzH9AXAtY/V/WbI0jXoe6jexOplTIa1g3iv6dmjJ/fQGTM9oRHenMWVCvNKvixx9/TPfu3XG5XEyePJmOHTvy4Ycf8vHHH9OlSxd27NjB5MmTadq0qU9yaKGDNQLm9setp0vffgim9IZRs6GJfRP1K+UUI9OTWLzpAMu3HuLu7on+3VkVZ9KBdsstt7BkyZIrvr5z505+9KMfkZmZ+f0Zfk2E7zX0ynTKgrFLoewCTB8Ae1bbnUipkHdj63iSG9ZhTq7zb45WV9u2bfnlL3/JM88845PP00K/XGIqPLjSmjZg5lDY/JbdiZQKaS6XMDQlkc93F1F44pzdcYLOpEmTWL16NXv27KnxZ2mhVyYuCcYth6SeMP9B+OS/dQSMUjWQleLBGFiw3plPjlY2TW6vXr0qvdxy+eu1a9emsLCQVq1a1TiHFvqV1G4A9823LsOsfBLe+XcoL7M7lVIhKalhHXq0asi89Tphlz9poV9NRC0YOg1u/inkvgyz74ULOsezUtdiWMp3E3adsDuKY2mhV8XlgjuehoHPwo5l8EoGnHL+o8xK+VpG56ZER7qY54fLLk4867+WY9JC91aPCTByFhTlW8MaC9fbnUipkBIbHcmAjk1ZsukAxSW+u3wZHR1NUVGRo0rdGENRURHR0dHV+n06Dr06bsiA8e/BrGzrTD3zJeiYaXcqpUJGVqqHtzceYOX2bxjcpblPPtPj8VBQUHDJE5hOEB0djcdTvXVZtdCrq0lHmPAB5IyGtx6Ak4Vw08PWw0lKqau6+bpGNKsfzby8Ap8VemRkpE9GiDiBXnK5FnUbwf0LocMQeO9xWD5ZR8Ao5QW3S7i7eyKrdx3l8Oliu+M4jhb6tYqMhmEz4MaH4Z8vwez74PwPx6IqpS6VleKhrNywcIMOLvA1LfSacLlgwB9gwDOwc5k1XcAJfbxZqau5vnEMXZPimLe+wFE3MoOBFrov3DgJRr0FJ76Gqb1h/1q7EykV1IalJPLlodNsOxDe86T7mha6r7Tpa80BExUDr94JXyyyO5FSQevOrs2JcvtnTHo400L3pYQbrFJv2hnm3A///IfdiZQKSnF1oujTvjGLNh6gpKzc7jiOoYXua3Ubwf2LoN0gWPYf8O7jOgJGqUpkpXgoOnOBD3c4a/y4nbwqdBEZICI7RCRfRCZX8n59EVksIptEZJuIjPV91BASVQfueQ16ToLPn4dZI6FYrxUqVdFtNyTQKCaKuXk6kMBXqix0EXEDLwAZQAdgpIhcvpTPw8AXxpiuQC/gLyIS5eOsocXlhoxnYNBfIX8lvNwPjtV8vmOlnCLS7WJoiodV2w9z5PR5u+M4gjdn6D2AfGPMbmPMBSAHGHLZNgaIFREBYoBjQKlPk4aq9PFw3wI4fQim9YGCXLsTKRU07klLorTcMDdPb476gjeFnghU/Jmo4OJrFT0PtAcOAFuAnxljfnCnQ0QmikiuiOQ6bd6Fq2p9Gzy4CmrFwozB8OU7didSKihc3ziGHi0bMnvdPh2T7gPeFHplk5Rc/l++P7ARaA50A54XkXo/+E3GTDHGpBlj0hISEqodNqQ1uh7Gr4TG7a151ddOtTuRUkEhu0cSe4vOsmb3MbujhDxvCr0ASKrwvQfrTLyiscB8Y8kH9gDtfBPRQWIS4IEl0KY/LH0UVjwB5TpkS4W3gZ2bERsdQc66fXZHCXneFPo6oI2ItLp4ozMbuPypmX1AHwARaQLcAOz2ZVDHiKoLI16HtHHw6d9gwUQo1RtCKnxFR7rJ7J7Isq2HOHH2gt1xQlqVhW6MKQUeAd4FtgNzjDHbRGSSiEy6uNnTwM0isgVYBTxmjDnqr9Ahzx1hjX7p8wRseQtez4Kz+uOmCl/Z6clcKC1nwYZCu6OENLHrRkRaWprJzdURH2yeAwsfhnqJMGq29bSpUmFoyPOfcKHMsOxnt9gdJaiJSJ4xJq2y9/RJUbt1uQfGLIEL38K0vrBrpd2JlLJFVqqH7QdP8YVO2HXNtNCDQXJPmPA+xLWAN++BvBl2J1Iq4O7s0pxIt+iEXTWghR4s4pJh3HK4rjcs/hm8/zvQcbkqjDSoG0Wfdk1YuLFQJ+y6RlrowaRWDIzMge73weo/w9s/hlK966/CR1aqh6PfXmD1zjB68NCHtNCDjTsS7vo79H4cNs2CN4frxF4qbPS6IYH4ulF62eUaaaEHIxG47T9gyIuw9xN4JQNO6fqLyvki3S7u6taclV8c1jHp10ALPZh1Hw2j5sDxr60RMAc3251IKb/LSvFwoaycxZsP2h0l5GihB7vr+8DYpdavp/fXpe2U43VsXo92TWOZm6vzpFeXFnooaNYFJnwAjTvAnIs3THUEjHIoEWFEehKbCk6y/aDeP6oOLfRQEdsEHngHOt9jDWlc+AiUldidSim/yOyeSFSEi5y1OmFXdWihh5LIaBg6BW57DDa+Dm+OgPOn7U6llM/F1Ykio1NTFmwopLhE1+T1lhZ6qBGB3r+yhjbu/hBeGagjYJQjjUhP4lRxKcu26s1Rb2mhh6qU+63JvI7thim9oTDP7kRK+dRNreNpGV+HWWv15qi3tNBDWZt+MH4FRERZZ+pb5tqdSCmfsW6OJrN2zzG+OvKt3XFCghZ6qGvSwRoB07w7zBsPq5/VETDKMbJSE4lwCbPX6Vm6N7TQnaBuI7h/4cURME/Dkp9DWandqZSqscax0dzerjHz1xfohF1e0EJ3iohakPkP+NEvrOl3c0bBef0xVYW+YTphl9e00J3E5YK+T1jL2+WvsJ4sPaHjeFVo63VDYxrqhF1e0UJ3ovTxMPotOLEfpt4O+9fanUipaxYV4eKurjphlze00J3q+r7w4EqoFQszBukIGBXShqXqhF3e0EJ3soS28OAq8KRbI2A+/R8dAaNCUsfm9bihSSzz8vSyy9VooTtdnYZw3wLoOBRW/BaW/j8o10epVWgREbJSE9m4/4SOSb8KLfRwEFELsl6Gm38K66bC7Pvgwlm7UylVLXd3S8Ql6Fn6VWihhwuXC+54GjL+DDuWwqt3wpmjdqdSymuN60Vza9sE5q8vpKxcLx1WRgs93PScCCNmwjdbrVWQjubbnUgpr41IS+LQqWI+2nnY7ihBSQs9HLW/E8YshvOnYNrt1qyNSoWAPu2b0CgmSifsugIt9HCV1AMmvA+xzWHmUFg71e5ESlUpKsJFVqqH9788zOFTxXbHCTpa6OGsQUsY/541a+PSR2HZZB0Bo4LeiLQkysoNb+nN0R/QQg930fUg+0248SH45//CWw9AyTm7Uyl1Ra0TYujZqiFzcvdTrjdHL6GFrsDlhgF/hP5/gO2L4bUhcPaY3amUuqKRPZL5uugsa3YX2R0lqHhV6CIyQER2iEi+iEy+wja9RGSjiGwTkY98G1MFxE0Pw/BX4MBGmNobDn9pdyKlKjWgU1PqRUcwS+dJv0SVhS4ibuAFIAPoAIwUkQ6XbRMHvAjcZYzpCAz3Q1YVCB0z4YF3rAePXu4Hu1bYnUipH4iOdJPZPZF3tx3i5NkSu+MEDW/O0HsA+caY3caYC0AOMOSybUYB840x+wCMMTpINJQlpVsjYOJawJv3wD//YXcipX5gWGoSF0rLWbJFF0n/jjeFnghU/Lmm4OJrFbUFGojIhyKSJyL3V/ZBIjJRRHJFJPfIEZ2sPqjFJcG45dA2A5b9B7z3ayjXFWNU8OiUWI+2TWJ0KoAKvCl0qeS1y28tRwCpwCCgP/AbEWn7g99kzBRjTJoxJi0hIaHaYVWA1YqxnipNfxA++7s1Y2PpebtTKQVcnLArxcP6fSfYrRN2Ad4VegGQVOF7D3D5zzgFwHJjzBljzFFgNdDVNxGVrVxuGPgs9H0Kts2HmZlw7rjdqZQCILO7NWHX/PWFdkcJCt4U+jqgjYi0EpEoIBtYdNk2C4FbRCRCROoAPYHtvo2qbCMCP/q5NWNjwTp4WZe2U8Ghcb1obmmTwPz1BTomHS8K3RhTCjwCvItV0nOMMdtEZJKITLq4zXZgObAZWAtMM8Zs9V9sZYvOw6y51b89ZE3sdWCj3YmUIivVw4GTxXyuY9IRY9MKNmlpaSY3N9eWfasaOrwd3hgOZ4sg8x/Q4S67E6kwVlxSRvrvV9KvfRP+OqKb3XH8TkTyjDFplb2nT4qq6mvc3lrarklHmHMffPRnXdpO2SY60s3gLs1YtvUQ354vtTuOrbTQ1bWJbQJjlkCXEfDB72D+RCjVFdmVPbJSPJwrKWPZlvBeRFoLXV27yGjrksvtv4Etc+CNLCg+aXcqFYZSWzSgZXwd5q0P7zHpWuiqZkTg1kchcwp8/TlMHwAnw/svlQo8EWFoioc1u4+x/1j4rperha58o+sIuHeuVebT+sEhHeSkAiuzu/UA+4IN4TsmXQtd+U7rXjB2mfXrVzJ0aTsVUEkN63BT63jmry/ArtF7dtNCV77VtBM8uBLqe+D1LNj4pt2JVBjJSvWwt+gseV+H59PMWujK9+onWmfqLW6Gt38MK57Qib1UQGR0akqdKDfzwnQqAC105R+14+De+ZA2Dj79G8y+F87rBErKv+rWimBAp6Ys2XSAcxfCb31cLXTlP+5IGPRXyPgz7FxmjYA5Fd7jhJX/3ZOWxOnzpSwNwzHpWujKv0Sg50QY/RYc32PNAXNY521T/tOzVUNaN6rL7DBcnk4LXQXG9X1h7FIoL7Fma9zzsd2JlEOJCCPSk1i79xj5h8PrMp8WugqcZl2tETCxTWHm3ZD3qt2JlEMNTfEQ4RJmrwuvaZ610FVgxSXD+Peg1a2w+Kew/JdQFt4TKinfS4itRb8OTZi3vpALpeEzwkoLXQVe7TgY9Rb0nARrXoRZ2ToCRvnciPQkjp25wIovvrE7SsBooSt7uCMg4xkY/N/w1fswYyCcDp+/eMr/bmmTQGJcbXLC6LKLFrqyV9o4GJkDR3fBy33hyE67EymHcLuE4WkePsk/SuGJc3bHCQgtdGW/tnfAA+9AyTl4uR98/ZndiZRDZKV4MAYWhMm0ulroKjgkplgjYOomwGt3w7YFdidSDpDUsA49WzVk3vrCsJiwSwtdBY8GLa0RMM27w1tj4bO/69J2qsayUj3sOXqG9ftO2B3F77TQVXCp0xDuf9taePq9X1tDG3VpO1UDAzs3o3akOyxWM9JCV8EnsjYMmwG3/Dusfw1mZsKZIrtTqRAVU2HCruISZ0/YpYWugpPLBX1+C0OnQsE6mHY7HM23O5UKUVkpHk4Vl7Jyu7OHxmqhq+DW5R5rBMz509YImP1r7U6kQtBN18XTrH40c/OcfdlFC10Fv6R0GL/CesL01Tth+2K7E6kQ43YJmd0TWb3zCIdPFdsdx2+00FVoiL/OKvUmnWD2ffDJ33QEjKqWrFQP5Qbe3ujc1Yy00FXoqNsIxiyGjnfDyies5e1Kz9udSoWI6xJi6JYUx7w8545J10JXoSWqDgx7BXr9CjbNsi7B6AgY5aWsVA87vjnNtgOn7I7iF1roKvSIQK/HYPgMOLjJull6bLfdqVQIuLNLM6LcLsfeHNVCV6GrYybcvwjOHYNp/aAgz+5EKsjF1Ymib4fGLNp0wJHzpGuhq9CW3NO6WRpVF2YMgi+X2p1IBbmsFA/Hzlzgwx2H7Y7ic14VuogMEJEdIpIvIpOvsl26iJSJyDDfRVSqCo3aWBN7NW4Hs0fDuml2J1JB7Na2CTSKiXLkVABVFrqIuIEXgAygAzBSRDpcYbtngHd9HVKpKsU0th5Aur4fvPPvsOIJKHfej9Sq5iLdLoZ0S+T9Lw9z7Iyz5gny5gy9B5BvjNltjLkA5ABDKtnuJ8A8wHk/x6jQEFUXst+E1LHw6d/grTFw4YzdqVQQuictiZIyw3yHnaV7U+iJwP4K3xdcfO17IpIIZAIvXe2DRGSiiOSKSO6RI0eqm1WpqrkjrGXt7vi99UTp9AFw0ll/aVXN3dA0lu7JceSs2++oMeneFLpU8trl/wX+BjxmjLnqVGbGmCnGmDRjTFpCQoK3GZWqHhG4+REYNQeO7YGpt8OBDXanUkEmOz2J/MPfkvf1cbuj+Iw3hV4AJFX43gMcuGybNCBHRPYCw4AXReRunyRU6lq1vQMeXAHuWvDKINj5nt2JVBAZ3KU5daPc5KzbX/XGIcKbQl8HtBGRViISBWQDiypuYIxpZYxpaYxpCcwFHjLGvO3ztEpVV+P2Vqk3uh5mZUPeq3YnUkGibq0I7uqWyJLNBzhVXGJ3HJ+ostCNMaXAI1ijV7YDc4wx20RkkohM8ndApWostqk1Aua63tYKSCt+C+XOXuhAeSc7PYniknIWbrz8okNoErtuCKSlpZnc3Fxb9q3CVFkJLHsMcl+GthmQNRVqxdqdStnIGMPA5z7B7YIlP7nF7jheEZE8Y0xaZe/pk6IqfLgjYfBfYeCzsOs9eLk/nHDO9VNVfSJCdnoSWwtPsbXwpN1xakwLXYWfHhPg3rlwcj9M6wsHN9udSNloSLfmRLldjnhyVAtdhafrbodx74LLDa9kQP4quxMpm8TViaJP+8Ys2niAkrLQfrpYC12FryYdrDlgGrSEN++BvBl2J1I2yUrxUHTmAh/uCO0HHrXQVXir1xzGLoPWvWDxz2D5L6Gs1O5UKsBuuyGB+LpRzAvxedK10JWKrgcjZ8OND8GaF2HWCCgO/RtkynvfTdi16stvOB7CE3ZpoSsF1hwwA/4Idz4Huz+E6Rlw0rmLCasfykpNpKTMsHhz6I5J10JXqqLUMTB6LpzYZ42A+Wab3YlUgHRsXp92TWND+rKLFrpSl7uuN4xbBhhrtsbdH9mdSAXIsFQPmwpOsuub03ZHuSZa6EpVpmlnawRMvUR4PQs2zbY7kQqAu7snEumWkJ2wSwtdqSup74FxyyH5RlgwET7+Czho7mz1Q41iatGvQxPmry/gfGnozfejha7U1dSOg3vnQefhsOo/YdFPoDR0R0GoqmWnJ3P8bAnvbfvG7ijVpoWuVFUiakHmFLj1/8GGmTDzbjhTZHcq5Sc/ur4RiXG1mR2Cl1200JXyhssFt/8ahk6DglyY2hsOf2l3KuUHLpcwIj2JT/KPsq/orN1xqkULXanq6DIcxi6FknMw/Q7Y+4ndiZQfDE/z4BKYnbvP7ijVooWuVHV50qwRMDFNYGYmbJlrdyLlY83q16b3DY15K7eA0hCasEsLXalr0aCFNVtjYhrMGw+fPqcjYBxmRHoSh0+f56OdoTNhlxa6UteqTkO4bwF0zIQVv7FWQ9Kl7Ryjd7vG1oRdITRPuha6UjURGQ1Z0+GmR2DtP2DO/db1dRXyIt0u7urWnJVfHObE2dAYqqqFrlRNuVzQ//eQ8V/w5TvwykA4ddDuVMoHslI8XCgrZ/Hm0Pjz1EJXyld6/itkvwlHdsDU2+HABrsTqRrq2Lwe7ZrGMjdEJuzSQlfKl9oNhPHvWUvbTc+ALxbanUjVgIiQleJh0/4T5B/+1u44VdJCV8rXmnaCCe9bE3zNGQOfv2B3IlUDQ7o3x+2SkLg5qoWulD/ENIYxi6D9YHj3V9bSduWhM55Z/Z/GsdHc2qYRC9YXUlYe3ENTtdCV8pfI2jD8Vej5Y2tpuzn3wYUzdqdS1yAr1cOhU8V89tVRu6NclRa6Uv7kckPGn6D/H2HHUpjeH04G/4/u6lJ92zehXnRE0K9mpIWuVCDc9BCMmgPHv4Ypva0JvlTIiI50M7hrc5ZvO8Tp4hK741yRFrpSgdKmH4xfAVF1YMYga8y6ChlZKR6KS8pZtuWQ3VGuSAtdqUBq3A4eXAVNOkLOaFg71e5EykspyXG0blSXuUE82kULXalAq9sIxiyBtgNg6aPw7uM6B0wIEBGyUj2s3XOM/ceCc550rwpdRAaIyA4RyReRyZW8P1pENl/8+kxEuvo+qlIOElUHst+A9Anw+fOQMwrOh+ZK8+Eks3siIgTtmPQqC11E3MALQAbQARgpIh0u22wPcJsxpgvwNDDF10GVchyXGwY9CwOfhV0r4OU74Pheu1Opq2geV5ubr4tn3voCyoNwTLo3Z+g9gHxjzG5jzAUgBxhScQNjzGfGmOMXv10DeHwbUykH6zHBWoj6VCFM6wuFeXYnUleRleJh/7FzrN17zO4oP+BNoScCFVdLLbj42pWMB5ZV9oaITBSRXBHJPXIkdCaNV8rvrutt3SyNrA0zBsOO5XYnUlcwoFNTYmtFMCcIF5H2ptClktcq/VlDRHpjFfpjlb1vjJlijEkzxqQlJCR4n1KpcNCoDYxfCY3aQs5IyJ1udyJViTpREdzVrTnvbDnIyXPBNSbdm0IvAJIqfO8BDly+kYh0AaYBQ4wxRb6Jp1SYiW0CD7wD1/eFJf8Gq/5Tl7YLQtnpyZwvLWfhxkK7o1zCm0JfB7QRkVYiEgVkA4sqbiAiycB84D5jzE7fx1QqjNSKgexZkDIGPv4LLPhXKA2NFXPCRWdPfTo2r8estfsxQfQPbpWFbowpBR4B3gW2A3OMMdtEZJKITLq42W+BeOBFEdkoIvpcs1I14Y6AO/8Hbv81bJ4Nr8ZpKRkAAAnNSURBVA+Fs8F3Ey6cZfdIZvvBU2wpPGl3lO+JXf+6pKWlmdxc7X2lqrQpBxb9BOp7YORsSGhrdyIFnCouocfvV5LZ3cMfh3YO2H5FJM8Yk1bZe/qkqFLBrms2jFkMxaesYY35q+xOpIB60ZEM6tycRRsLOXO+1O44gBa6UqEh+UaY+AHEJcEbw2H9a3YnUkB2jyTOXCjjnS3BsYi0FrpSoSIuGcYth9a9rEswH/xBR8DYLK1FA1o1qhs086RroSsVSmrFwqjZ0O1e+OgZWPiwjoCxkYgwtHsi/wySCbu00JUKNe5IGPI89PolbHwDXhsCZ4J7aTQny0yxHpyfv97+Mela6EqFIhHoNRmyXrbmfpnaG775wu5UYcnToA43tY5n/oYC28eka6ErFco6D4Oxy6D0vDVb41fv250oLGWlevi66Cy5Xx+vemM/0kJXKtR5UmHCB9ZN0zeGw8Y37U4UdjI6NaVOlNv2m6Na6Eo5Qf1EGLcMWvwLvP1j+Oi/dARMANWtFcGATk15Z/NBikvsW31KC10pp4iuD6PnQpds+OD3sPinUBYcD7yEg2EpHk6fL+XdbfYtIq2FrpSTRERB5ktwy6PWw0c5I+H8t3anCgs3to4nqWFtctbaN0+6FrpSTiMCfX4Dg/8G+SthxiA4FRxPMjqZyyWMSEvi891F7D16xp4MtuxVKeV/aWNhZA4U5VvDGgvX253I8YalJuESmJ1rz1m6FrpSTta2P4x7F1wR8MpA2Drf7kSO1rR+NLe3a8zcvAJKysoDvn8tdKWcrmkna1hjs64wd6y1aIaOgPGbEenJHDl9nve/PBzwfWuhKxUOYhJgzCLoNMxa1u6dX+gIGD/pfUMCTerVImftvoDvWwtdqXARUQuGToV/+bm1APXse+GCPTfvnCzC7WJ4ahIf7TzCgRPnArpvLXSlwonLBf2egoHPwq534dU74dsjdqdynHvSkig3MDfAT45qoSsVjnpMgBGvWxN6vdwXir6yO5GjJMfX4cbWDZm/PrATdmmhKxWu2g2ylrY7f9pa2u7rz+1O5ChZKR72Fp0lL4ATdmmhKxXOktJh/Aqo3cC6/LLhDbsTOUZG52bUjnQzb33gLrtooSsV7uKvgwmroOW/wMKH4N3Hody+CaacIqZWBBmdmrJkU+Am7NJCV0pZZ+ij50L6BPj8eZhzP5QEdoSGE2WlWhN2vffFNwHZnxa6UsrijoRBz8KAP8GX78Crd8GZIrtThbSbWsfTvH50wOZJ10JXSl3qxh/DPa/CwU3wcj8dAVMDLpcwNMXDx7uO8M2pYv/vz+97UEqFng5DrCdLzx2DaX1gz2q7E4WsrFQP5QYWbPD/ItJa6EqpyiXfCBPeh7qNYWam9XSpqrZWjeqS3rIBc9bt9/uYdC10pdSVNWwND66A1r1hyb9dHAET+FkEQ112ejK7j55h7Z5jft2PFrpS6uqi68Oo2dDjX60RMHPHQon/rwc7ycDOzYiNjiBnnX/nSddCV0pVzeWGjGfgjt/BF2/DzLvhrH/PNp2kdpSbu7slsnTLQU6eLfHbfrTQlVLeEYGbfwLDXoHCPOtm6ZEddqcKGSPSkzhfWs7bG/13c9SrQheRASKyQ0TyRWRyJe+LiDx38f3NIpLi+6hKqaDQaSiMWfJ/c8DsWml3opDQKbE+nRPrM2vtPr/dHK2y0EXEDbwAZAAdgJEi0uGyzTKANhe/JgL/6+OcSqlgktzTGgETlwxvDoe542HHMig9b3eyoJbdI4kvD51mU8FJv3x+hBfb9ADyjTG7AUQkBxgCfFFhmyHAa8b6Z2eNiMSJSDNjjC41rpRTxSVb65Wuegq2vAVb50Kt+lCvmd3Jgla2MfSsdYady+6m28Tf+/zzvSn0RKDirdkCoKcX2yQClxS6iEzEOoMnOTm5ulmVUsGmVgwM/DP0/wPs/hC2L4biE3anClpuoGHdC/Tp2skvn+9NoUslr11+AcibbTDGTAGmAKSlpekqtUo5hTsS2vSzvtRVNfTjZ3tzU7QASKrwvQc4cA3bKKWU8iNvCn0d0EZEWolIFJANLLpsm0XA/RdHu9wInNTr50opFVhVXnIxxpSKyCPAu1iXgKYbY7aJyKSL778ELAUGAvnAWWCs/yIrpZSqjDfX0DHGLMUq7YqvvVTh1wZ42LfRlFJKVYc+KaqUUg6hha6UUg6hha6UUg6hha6UUg4h/l5B44o7FjkCfH2Nv70RcNSHcUKBHnN40GMODzU55hbGmITK3rCt0GtCRHKNMWl25wgkPebwoMccHvx1zHrJRSmlHEILXSmlHCJUC32K3QFsoMccHvSYw4Nfjjkkr6ErpZT6oVA9Q1dKKXUZLXSllHKIoC70cFyc2otjHn3xWDeLyGci0tWOnL5U1TFX2C5dRMpEZFgg8/mDN8csIr1EZKOIbBORjwKd0de8+H+7vogsFpFNF485pGdtFZHpInJYRLZe4X3f95cxJii/sKbq/QpoDUQBm4AOl20zEFiGtWLSjcA/7c4dgGO+GWhw8dcZ4XDMFbZ7H2vWz2F25w7An3Mc1rq9yRe/b2x37gAc86+AZy7+OgE4BkTZnb0Gx3wrkAJsvcL7Pu+vYD5D/35xamPMBeC7xakr+n5xamPMGiBOREJ5hdoqj9kY85kx5vjFb9dgrQ4Vyrz5cwb4CTAPOBzIcH7izTGPAuYbY/YBGGNC/bi9OWYDxIqIADFYhV4a2Ji+Y4xZjXUMV+Lz/grmQr/SwtPV3SaUVPd4xmP9Cx/KqjxmEUkEMoGXcAZv/pzbAg1E5EMRyROR+wOWzj+8OebngfZYy1duAX5mjCkPTDxb+Ly/vFrgwiY+W5w6hHh9PCLSG6vQf+TXRP7nzTH/DXjMGFNmnbyFPG+OOQJIBfoAtYHPRWSNMWanv8P5iTfH3B/YCNwOXAesEJGPjTGn/B3OJj7vr2Au9HBcnNqr4xGRLsA0IMMYUxSgbP7izTGnATkXy7wRMFBESo0xbwcmos95+//2UWPMGeCMiKwGugKhWujeHPNY4E/GusCcLyJ7gHbA2sBEDDif91cwX3IJx8WpqzxmEUkG5gP3hfDZWkVVHrMxppUxpqUxpiUwF3gohMscvPt/eyFwi4hEiEgdoCewPcA5fcmbY96H9RMJItIEuAHYHdCUgeXz/graM3QThotTe3nMvwXigRcvnrGWmhCeqc7LY3YUb47ZGLNdRJYDm4FyYJoxptLhb6HAyz/np4EZIrIF63LEY8aYkJ1WV0RmAb2ARiJSADwBRIL/+ksf/VdKKYcI5ksuSimlqkELXSmlHEILXSmlHEILXSmlHEILXSmlHEILXSmlHEILXSmlHOL/A0Y63sx/pr+IAAAAAElFTkSuQmCC\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(df_idea.threshold,df_idea.tpr,label='TPR')\n",
    "plt.plot(df_idea.threshold,df_idea.fpr,label='FPR')\n",
    "plt.legend()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T01:57:59.861452600Z",
     "start_time": "2023-12-09T01:57:58.165551900Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## ROC曲线"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "outputs": [
    {
     "data": {
      "text/plain": "<matplotlib.legend.Legend at 0x2677edbd340>"
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": "<Figure size 360x360 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATwAAAEvCAYAAADYR30zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deVxVdf7H8deXHREQERVB1Nw1d3LLxNytzKmmKZsWayaz1PYmJ/upWdmG7TpmatrUZKVlNmVmmpprahoKiqEioiD7vt3l+/sDcsgwL3Lh3OXzfDx8yOUe7n0fhTffs32P0lojhBDuwMPoAEII0VCk8IQQbkMKTwjhNqTwhBBuQwpPCOE2pPCEEG7Dy6g3btasmW7btq1Rby+EcFH79u3L0lqH1fScYYXXtm1b9u7da9TbCyFclFLq5IWek01aIYTbkMITQrgNKTwhhNuQwhNCuA0pPCGE25DCE0K4DSk8IYTbuGjhKaWWKaUylFKHLvC8Ukq9qZRKUkrFKaX62j+mEELUnS0jvOXA2D94fhzQserPZOBfdY8lhBD2d9ErLbTWW5VSbf9gkQnA+7py6uRdSqkmSqlwrXWanTL+T1EmHF0H2mr3lxZCNBytwWy1UmHWmCxWKizWc39XmK2YzJpScwVflcXz9yEziIzqbJf3tcelZRHAqWqPU6s+97vCU0pNpnIUSFRUVO3faevL8OPiSwophHAcCvCu+lOTEqV4tHkztjfyJ+rQau6Jesou72uPwlM1fK7GG2VorRcDiwGio6NrfzONUz9C1CD487Jaf6kQ4n9KTWZyi01kF1eQW1xBbkkFOcUV5BSbyCkuJ7fEdO5z+aUmbLn1jZ+3J/7eHvh6e/7mY38vT/x8PKr+9sTPywM/7/M+Pvc1nphVCfN/eYmkoiQe7foAt/W83W7rbY/CSwVaV3scCZyxw+v+lqkMzh6CQdMgqJXdX14IR2a2WCk1WSgzWSkzWSgzWc49Lq16fO7zFRbKzNaqvy3kl5jIKqogq6ic7OJysosqKKmw1Pg+gX5eNGvsS7PGgYQ29+Wyxj5Vjyv/DgnwIcDHC38fD/x9vPD39qSRjye+Xh4oVdPYp/YmfzuZEyUnmD9sPiPbjLTLa/7KHoW3FpimlFoJDADy62X/XXocWM0QGW33lxbC3ixWTWZhOWn5paTnl5GWX0Zafilp+WXnHmcWlmO1Yeikq17vUvh4eRDs732utNqGNqJZY19Cq5VYaNXfTQN88PP2vKT3sZcDGQfYmbaTx6Mft3vZgQ2Fp5T6CBgGNFNKpQKzqdr01lovAr4GrgGSgBLgbrunBEitmkoqQgpPGE9rTU5xBSk5JaTklHAyu/LPqZwSUnNLOFtY/ruS8vXyIDzYj/Bgfwa0a0pYoC9enraNiny9Kjf3/Lw9frP55+/z+8/9+revlwceHvYZdTWURXGLCPEN4eZON9fL69tylHbiRZ7XwFS7JbqQ0/sgKAKCwuv9rYSoMFs5W/C/kVn1UdqpnFJSckooKjf/5mtaBvkR1bQRA9uH0irYn5bBfucKLjzYjyaNvO222eeKDmUdYvvp7TzU9yEaeTeql/cwbALQWkuPg/DeRqcQLqSwzMTRs4UcTiskKaOIM3mlVaVWRlZR+e+WD/T1okWwH61D/OnfrilRTRsR1bQRbUIb0bppI8M3B53dO3HvEOwbzMQufzjGqhPnKTxzOfgGGp1COCGzxUpydglH0gtITK8suCPpBaTmlp5bprGvFxFNKkdl3VsFnRuV/TpKaxnsR6DfhU6iEHWVmJPI5lObmdp7KgHeAfX2Ps5TeELUwGSxkllYztmCMs4WlJNRWMbZgjIyCso5W1jO2fwykrOLKTdXnqzu6aG4rFkAfaJCmNg/ii4tA+kSHkSrYD/Z3DRIhaWCV/a+QmPvxtzW9bZ6fS8pPOEQykwWzuSV1vicVcPZgjKOZxZxPKuY5Kxi0gvKySgoI7u44nfLe3oomgf60jzIj6jQRgzt1IwuLYPoEh5I+7DGsunpQIpNxTz0/UPsTtvN7EGzCfIJqtf3k8IThimpMLM5MZOvDqbx/ZGMC54bVp2/tydtmwUQ0cSPPlFNaBHoR/MgX1oE+dI80I8WQX40DfDB08mOTrqjvLI8Htj4AAnZCTw/5Hmub399vb+nFJ5oEKUVFk7nlXI6r5TU3BK2J2Xx/ZFMSk0WmjX24YY+EfRrE3LBogoL9OWyZo1pEeQrm54uwGQxcd9395GUm8Srw15leNTwBnlfKTxRb87klfL6d0f57nAGOedtejZr7MtN/SK4pkc4A9qFyojMzbx78F0SshMatOxACk/Ug/wSEwu3JLF8ezJaw3U9w2nfvDERTfxp1cSfiBB/Wgb5Scm5qYTsBN6Ne5frLruOUW1GNeh7S+EJuykzWXh/ZzILvj9GQZmJG3pH8MioTrRuWj8nkQrncSDjAF8e+xKNZnfabkL8QpjRf0aD55DCE3VmsWrW7D/NqxuOcjqvlJhOYfxjbGe6two2OppwAGlFaTyw8QHMVjONvBrh5+XHc0OeI9i34b8/pPBEnfzwSybPf3WYI+mF9IgI5pU/92Rwh2ZGxxIOwmw1M+OHGVisFlaNX0VU0CXMg2lHUnjikq2PT2fKB/toHdKINyf24boe4U53sbqoX+/GvctPGT8xb8g8w8sOpPDEJdqfkstDK/fTM7IJH907gEY+8q0kfisxJ5FFcYsYf9l4xrcfb3QcQG7TKC7BL2cL+fuKvYQF+rL0rmgpO/E7Wmvm751PoE8gT/Z/0ug450jhCZtVmK28vekXrn1rGxpYfnd/mjX2NTqWcEDbTm9jZ9pOpvScYsjBiQuRX83CJvtTcpmx+iCJZwu5tkc4s6/vRvNAP6NjCQdkspiI3RtLVGAUt3S+xeg4vyGFJ/5QUbmZ2PWJrNiZTMsgP5bcGc3Ibi2MjiUc2Dtx73A8/zhvD38bb0/HmlJLCk/8jsWq+fFEDmt/Ps3XB9MpKDNx58A2PD6ms8wJJ/5QQnYCSw4u4fr21xPTOsboOL8jhSd+40xeKRPf3cXJ7BIa+XgyulsL7hrclj5RIUZHEw4uqzSLmdtmEuoXyj+u+IfRcWokhSfOKTNZmPLBPrKLKnjj1t6M7tYSfx+ZO05cWHpxOutOrGNjykbiMuMAeHvE2w51oKI6KTwBVJ5G8PSaQ8Sl5rP4jn6M7t7S6EjCwVm1lTvW3UF6cTpdm3Zlau+pjGwzkvZN2hsd7YKk8AQAH+w6yap9qTw4oqOUnbDJqcJTpBenM3PATG7tcqvRcWwi5+EJ9iTn8MyXCYzo0pyHR3Q0Oo5wEgnZCQD0bu48dxOUwnNzZSYLj35ygMgQf169pbdcCytsorVm9S+raezd2KE3Yc8nhefm3t16nFM5pcy7oQfB/nLKibDNp0c/ZXfabh7p9wjeHs7zfSOF58bO5JWyYHMS1/RoKVM6CZudKjxF7N5YBoYP5OZONxsdp1ak8NzY818fBuCpa7oanEQ4i7yyPB747gG8PLyYO3iu091QSQrPTe08ls1XcWncH9OByBCZgl1cXG5ZLlM3TuVM0RnevPpNwhuHGx2p1uS0FDdktlh55st4IkP8uS/mMqPjCCdwPP840zZO42zxWV6JeYXoltFGR7okUnhuJr/ExL+2HONIeiGLbu+Hn7dcSSEuTGvNdynfMXvHbLw9vFk6ZqlTnYZyPik8N1BaYeG7w2f54sAZthzNwGTRjOnegjHdZdYTUTOrtvJ9yvcsilvEkZwjdGjSgbeGv0VkYKTR0epECs9Faa3ZcjSTLw6cYX18OiUVFloE+XLXoLZM6B3B5RFBTrfDWTSM7NJspm6cSnx2PFGBUTx35XNce9m1eHk4f104/xqI3zmZXcxTnx9ke1I2wf7eTOjdivG9WjGgXajc/Fr8oRJTCdM2TuNY3jGXKrpfuc6aCArLTKzYkcxbm5Lw8fTg2T9dzl+iI/H1kv104uJMVhOPbXmMhJwE3rj6DYa1HmZ0JLuTwnMBWUXlvLf9BO/vPElhmZkx3VvwzPWX0zJYpmAXttFaM3fnXLad3sacQXNcsuxACs+pFZebefv7JJZtO0GFxcrY7i25f1h7ekY2MTqacDIHsw6yJmkN9/a4l5s63WR0nHojheeEtNb8Ny6N5786THpBGTf2iWDq8A60D2tsdDThpNYnr8fbw5u7L7/b6Cj1SgrPyZzJK+Wfnx1ky9FMLo8IYuHtfekr06+LOrBqK9+e/JYrW11JoE+g0XHqlRSek9Bas/qn0zyzNh6zVTNnfDfuGNRWjrqKOovLjCO9OJ0H+zxodJR6J4XnBLKLyvnnZwf5NuEs/ds1JfbPvYgKletfRd1llGTwf9v/j8bejV32QEV1Nk0eoJQaq5RKVEolKaVm1PB8sFLqS6XUz0qpeKWUa+8IaEAWq2byv/exOTGTmdd0ZeW9A6XshF1klmRyz/p7yCjJYOHIhS6/OQs2jPCUUp7AAmAUkArsUUqt1VonVFtsKpCgtR6vlAoDEpVSH2qtK+oltRtZtu0E+07m8upfenFjX+e+rEc4jvzyfCZvmExGSQaLRy126utja8OWEV5/IElrfbyqwFYCE85bRgOBqvJapcZADmC2a1I3lJheyCvfJjKqWwtu6BNhdBzhIopNxdz/3f2kFKTw1vC33KbswLbCiwBOVXucWvW56t4GugJngIPAQ1prq10Suqkyk4XpH/1EkJ8X827oIde9CruosFTw0KaHSMhO4JWYVxgQPsDoSA3KlsKr6SdNn/d4DHAAaAX0Bt5WSgX97oWUmqyU2quU2puZmVnrsO7kua8SOHq2iPl/6U1YoK/RcYQLsGorT217it3pu3n2ymcZHjXc6EgNzpbCSwVaV3scSeVIrrq7gc90pSTgBNDl/BfSWi/WWkdrraPDwsIuNbPL+zY+nQ92pTB56GXEdJJ/J1F3Wmte2fMK65PX80i/RxjffrzRkQxhS+HtAToqpdoppXyAW4G15y2TAowAUEq1ADoDx+0Z1F1orXn9u1/o2Lwxj4/ubHQc4SKWxy/ng8MfcHvX27m7u/ueRHHRwtNam4FpwHrgMPCJ1jpeKTVFKTWlarFngcFKqYPARuBJrXVWfYV2ZYdOF5CQVsCdg9rg4yW3HBF19++Ef/PqvlcZ03YMT1zxhFvvD7bpxGOt9dfA1+d9blG1j88Ao+0bzT2t3JOCn7cH1/eWo7Ki7t6Pf59X9r7CyKiRvHDVC3go9/4lKldaOJCicjNfHDjDNZeHy02xRZ2tiF9B7N5YRrUZxUtDX3KqG2bXFyk8B7J6XypF5WbuGNTG6CjCyS0/tJz5++Yzus1oXhz6opRdFfce3zoQq1WzfEcyvVs3oY/MfiLq4JPET5i/bz5j246Vkd15pPAcxJZfMjmRVczdV7Y1OopwYgnZCbz444sMiRjCC1e94FL3o7AHKTwHUFph4ZVvEmke6Mu4y53vbu7CMRSbinliyxOE+IUwb8g8KbsayL+IwbTWzPgsjsPpBSy9K1pORRGXJLMkk39s/QepRaksHb2UED/ZLVITKTyDLfnhBF8cOMMTYzozvIvcGFvU3o7TO/jntn9Sai7l+SHPE90y2uhIDksKz0DfH8nghXWHuaZHSx4Y1t7oOMLJ5Jfns+jnRXx4+EPaN2lPbEws7ZvI99EfkcIzSHG5mUc+OUDX8CBib+7l1me/i9oxWUysTFzJop8XUVhRyF86/4XHoh/D38vf6GgOTwrPIJ/uPUVeiYmld11BIx/5bxC2KTGVMHnDZH7O/JlB4YN4LPoxOjeVa65tJT9pBqgwW1m89Tj92oTQr43sXBa2KbeU8+CmBzmUdYiXrnqJce3GyZZBLckhQQN8svcUZ/LLeHBER6OjCCdhspp4fMvj5+ayu+aya6TsLoEUXgMrLjfz1qZf6BvVhKEdmxkdRzgBrTVzdsxh86nNPDXgKbedy84epPAa2ILvkzhbUM7Ma7vJb2hhk7f2v8XaY2t5oNcDTOwy0eg4Tk0KrwGdLShjyQ8nuLFvhOy7EzZZeWQl7x58l5s63sSUXlMu/gXiD0nhNaBNRzKosFiZEiPnSomL25q6lXm75zGs9TCeHvi0bBHYgRReA9qSmEmrYD86Nm9sdBTh4PLK8pi1fRYdQzry8tCX5bpYO5F/xQZisljZnpTFdb3C5Te1uKh5P84jvzyfd0a9IycU25GM8BrIgu+TKCw3M1ZmQxEXseHkBtadWMd9ve6Tk4rtTAqvAew7mcObG3/hxj4RcttF8Yf2pO9h5raZdAvtxt96/M3oOC5HCq+eFZSZeGjlASJC/HlmQnej4wgHtjttN1M3TiU8IJwFIxbITMX1QAqvns3+Ip60/DJev6UPgX7yDSxqtuHkBu7/7n4iGkewdMxSmvnLSen1QQqvHq3Ykczn+08zfXgHOe9OXNDqo6t5fMvjdA/tzvKxy6Xs6pEUXj1ZsSOZ2WvjGdm1BdOu7mB0HOGg9qTv4ZmdzzC41WAWj15MsG+w0ZFcmpyWYmdWq+bVDUd5+/skRnVrwYLb+uLlKb9XxO/lluUyY+sM2gS1YX7MfDn9pAFI4dnZ+zuTefv7JCb2b83cCZfjLWUnaqC1ZtaOWeSW5/L2iLdp5N3I6EhuQX4a7chq1azYeZJ+bUKYd0MPKTtxQWuS1rD51GYe7fcoXUO7Gh3HbchPpB3tOJbNiaxi7hjYRq6mEBdUai7lrf1v0TusN3/t+lej47gVKTw7+mDXSZoG+DCuR0ujowgH9uHhD8kszeSRfo/IL8YGJvvw7CQ9v4wNh8/y96va4evlaXQc4YAKKwpZcnAJ/074NzGRMfRt0dfoSG5HCs8OtNa8sfEoFqvmr/3bGB1HOBiL1cLqX1az4MACcspyGH/ZeB6LfszoWG5JCs8OFm05zkc/nmLy0MuICpWjbeJ/9mfsZ97ueRzJOUJ0i2gWjlxI91C5xNAoUnh1tHpfKi99c4TxvVoxY2wXo+MIB3G2+Cxv7n+TtcfW0jKgJfNj5jOqzSjZZ2cwKbw62J6UxZOr4xjcPpTYm3vi4SHfzO4suzSbjxM/ZvOpzRzJOYKnhyd/7/F37u1xr5xn5yCk8OrgpW+OEBnizzt39JMDFW7sdNFplh9azudJn1NhqaBP8z5M7T2Vce3GERUUZXQ8UY0U3iU6dDqfuNR85ozvJrOguKlScymv73udjxM/RinF9e2vZ1L3SbQLbmd0NHEBUniX6MPdKfh5e3BD30ijowgDxGXGMXPbTJILkrml8y3c2+NeWgS0MDqWuAgpvEuQX2pizf7TjO/ZimB/Gd25mz3pe5i8YTLN/JuxdPRS+of3NzqSsJEU3iVYvS+VUpOFuwa3NTqKaGDJ+ck8svkRWge25t/j/i3TOTkZubSslvJLTSzcfIzoNiFcHiHf7O4kuzSb+7+7H0/lyYLhC6TsnJBNhaeUGquUSlRKJSmlZlxgmWFKqQNKqXil1Bb7xnQcr6w/Qk5xOXOul5NH3UmpuZTpm6aTVZrFW8PfonVQa6MjiUtw0U1apZQnsAAYBaQCe5RSa7XWCdWWaQIsBMZqrVOUUs3rK7CR9p3M5cPdKdw9uJ2M7txEhaWCNUlrWHJwCenF6bx29Wv0DOtpdCxxiWzZh9cfSNJaHwdQSq0EJgAJ1Za5DfhMa50CoLXOsHdQo5WZLMz8/CAtg/x4dHQno+OIemC2mtmYspEDGQdIKUwhpSCF1KJUzFYzPcN6MvfKuQwMH2h0TFEHthReBHCq2uNUYMB5y3QCvJVSm4FA4A2t9ft2SegA8ktNTH5/L0fSC3n3zmga+8qxHlehteZkwUl+OP0DHyR8wJniM/h7+dM6sDUdQzoyPGo4A8IHMCh8kFwW5gJs+cmt6X9Z1/A6/YARgD+wUym1S2t99DcvpNRkYDJAVJRznIFeWmHhlnd2ciyziDdu7c2obnKulSsoqCjguZ3Pse3MNgorCgHo27wvM/rPIKZ1DB5Kjue5IlsKLxWovoc2EjhTwzJZWutioFgptRXoBfym8LTWi4HFANHR0eeXpkP6eE8KR9ILWXxHP0Z3l4k9XcGZojM88N0DnCw8yYT2E+gZ1pNeYb1o36S90dFEPbOl8PYAHZVS7YDTwK1U7rOr7gvgbaWUF+BD5Sbva/YMaoQKs5V3th7nirYhUnYuIiE7gakbp1JuLuedke/IScNu5qKFp7U2K6WmAesBT2CZ1jpeKTWl6vlFWuvDSqlvgDjACizRWh+qz+AN4fP9qaTll/HCjT2MjiLq4GjuUTac3MDBzIPsO7uPEL8Q3h33Lh1C5H7B7samve9a66+Br8/73KLzHr8CvGK/aMayWjWLthynR0QwMZ3CjI4jLlGJqYS71t1FibmEDk06ML79eKb0mkLzRi555pS4CDnceAHbj2VxIquYNyf2kaNzTuy7lO8oMhXx3pj3iG4ZbXQcYTA5FHUBK388RZNG3ozpLkdlndmapDVEBUbRr0U/o6MIByCFV4PsonK+TUjnxj6RMrGnEztVeIo96XuY0GGCjNIFIIVXo9U/pWKyaG7tL9dLOrP/HP4PisqJOYUAKbzf0Vrz0Y+niG4TQqcWgUbHEZdo5ZGVfHD4A27oeAMtA+SUIlFJCu88O49lcyKrmNsGOMeVIOL3vjz2Jc/vfp5hrYfx9MCnjY4jHIgcpa2m8obavxAa4MM1PcKNjiNqyWK18MHhD3ht32v0b9mf2JhYvD1kRmrxP1J41XybcJbdJ3J49k+X4+ctByucyemi08zcNpN9Z/cxvPVw5l01D19PX6NjCQcjhVelwmzlxXVH6NC8MROvkIMVzqTMXMbf1/+dvPI8nh/yPOMvGy9HZUWNpPCqrNl/mhNZxSybFI2Xp+zadCZLDy0ltShVbqgjLkp+sqncd/fJ3lO0axbA1Z3lkiNnklKQwrKDy7im3TVSduKipPCAL+PS2Hsyl3uGtJNNISdispj45w//xMfTh8ejHzc6jnACbr9JW1hm4rn/JtAjIpjb+supKM7ktZ9eIy4rjvkx8wlrJBM8iItz+8JbuPkYmUXlLL4zGk8PGd05i08SP+HfCf9mYpeJjG472ug4wkm4deFlF5WzYkcy43u2onfrJkbHETb6+MjHPLf7OWIiY2RTVtSKWxfe4h+OU2ay8OCIjkZHETb66MhHzNs9j2GthzE/Zj4+nj5GRxJOxG0PWuQWV/D+jpNc36sVHZo3NjqOsMGGkxuYt3seV7e+mldjXpWyE7XmtoV3OK2AUpOFm6PlJGNncDT3KDO3zaRnWM/KS8Y85ZIxUXtuW3h5pSYAmgbIKMHRZZRk8NCmhwjwDuC1Ya/JyE5cMrfdh5dfVXhNGslIwVGVmEpYkbCC9w69h8VqYemYpXIvClEnbl94wf5SeI7ocPZhpm2aRkZJBqPajOLhvg8TFSTnSYq6cevC8/ZU+MusKA5p4YGFmCwmVoxdQd8WfY2OI1yE2+7DyymqINjfWy4lc0CphalsSd3CXzr/RcpO2JXbFt6BU3l0DQ8yOoaowSeJn+ChPLi5081GRxEuxi0LL6OwjMSzhQy8LNToKOI8ZeYyPkv6jOFRw2kRILfIFPblloX36rdH8fRQjOkuN3dxNIt+XkR+eT4Tu0w0OopwQW5XeHuTc1i55xR/H9JOrrBwMKuOrmLpoaXc1PEmoltEGx1HuCC3KjyTxcrTaw7RKthPrp91MNtOb+O5Xc9xZcSVPD3waTmYJOqFWxXe6n2pHEkvZNb47gT4uu0ZOQ4ntyyXGT/MoEOTDsyPmY+Xh/zfiPrhNoVnsWre2XqcHhHBjOkuO8MdyRs/vUFRRREvXvUiAd4BRscRLsxtCu+bQ+mcyCrmgWHtZXPJgWw/vZ3Vv6zm9q630yGkg9FxhItzm22H9fHptAjyZbQcmXUIP2f+zL9+/hfbT28nonEE9/e+3+hIwg24TeGdySulXbMAmcbdYIezD/PG/jfYfno7Ib4hPNz3YW7tcqtsyooG4TaFl5ZfxoB2TY2O4baKKoqYt3seXx7/kmDfYB7p9wi3dr6VRt6NjI4m3IhbFJ7FqjlbUEbLYD+jo7ilM0VnmLpxKsn5yfzt8r9xT497CPKRy/pEw3OLwssqKsds1YQ38Tc6ituJz4pn6sapVFgqWDhyIYNaDTI6knBjblF4Z/JKAQgPkhFeQ9Ja84+t/8DX05dlY5ZxWZPLjI4k3JxbnJaSll8GQHgTKbyGtD9jPymFKUztM1XKTjgEtyi8zYkZ+Ht70jZUjgQ2pLXH1uLv5c/IqJFGRxECcIPCyy2u4IsDZ/hTnwi5nKwBlZpL+Sb5G0a3GS1HYoXDcPnC23D4LOVmK38dIPdDaEgbUzZSbCpmQocJRkcR4hybCk8pNVYplaiUSlJKzfiD5a5QSlmUUn+2X8S6+flUHoF+XnST2Y0bzNbUrbyy5xUiG0fSr0U/o+MIcc5Ft/GUUp7AAmAUkArsUUqt1Von1LDcS8D6+gh6qX5OzaNnZDAecoVFvSsxlTB/73w+OfoJHUM68vJVL+OhXH4jQjgRW74b+wNJWuvjWusKYCVQ03bKdGA1kGHHfHVSZrJwJK2QXpFNjI7i8ooqirh93e18evRTJnWfxMprV8pkAMLh2LIXPwI4Ve1xKjCg+gJKqQjgBmA4cIXd0tVRQloBZqumV2spvPqktebp7U9zPO84b494m6GRQ42OJESNbBnh1bQtqM97/DrwpNba8ocvpNRkpdRepdTezMxMWzNesrhTeQD0jAyu9/dyZ8sOLWNjykYe6feIlJ1waLaM8FKB1tUeRwJnzlsmGlhZNc9cM+AapZRZa72m+kJa68XAYoDo6OjzS9PuDp4uICzQl5ZyhUW92X56O2/uf5MxbcdwZ7c7jY4jxB+ypfD2AB2VUu2A08CtwG3VF9Bat/v1Y6XUcuC/55edEQ6ezqNHRLBM+FlPVh1dxfO7nqd9k/bMHTxX/p2Fw7to4WmtzUqpaVQeffUElmmt45VSU6qeX1TPGS9JcbmZpIwixl0ebnQUl2O2mondG8uHhz/kylZX8nLMy3JysXAKNl16oLX+Gvj6vM/VWHRa60l1j1V3CWkFWLXsv7Onoooitp3exphCupIAABf5SURBVMrElew7u487ut3Bo/0elZvuCKfhst+pP53MBaBHhBReXe1N38vSQ0vZnbYbk9VEU7+mzB08lxs63mB0NCFqxWUL75v4dLqFB9FcDljUyanCU0zfNJ1G3o24rcttDI8aTq+wXnh6eBodTYhac8nCO51Xyv6UPJ4Y09noKE7NZDUxY+sMFIr3x71PROMIoyMJUScuWXjrDqYBcG0POWBRFwsPLCQuK47YmFgpO+ESXPJCx68PptG9VRBtm8n8d5dq86nNLD24lJs63sSYtmOMjiOEXbhc4eWVVHDgVB4ju7YwOorTSsxJ5MmtT9IttBtP9n/S6DhC2I3LFd62pCysGoZ2CjM6ilPKKMlg+qbpNPZpzJvD38TfS258JFyHy+3D25KYSbC/N73k/LtaK6wo5P7v7ie/PJ/3xr5H80bNjY4khF25VOGVmSx8E5/OyK4t8PJ0ucFrvTJZTTy2+TGO5x1nwcgFdAvtZnQkIezOpVphfXw6hWVmbo6ONDqK03npx5fYmbaTWYNmMbjVYKPjCFEvXKrwPt5zitZN/RnYLtToKE7l818+5+PEj7n78rvl6gnh0lym8E7llLDjWDY392st07nXQrGpmNd/ep2+zfvycN+HjY4jRL1ymX14+6sm+5TTUWpnefxycspyeHv423L/CeHyXOY7PDmrGIB2crKxzTJLMlkRv4IxbcfQI6yH0XGEqHcuVXjhwX74+8hF7bb618//wmQ18VCfh4yOIkSDcJnCO55VTNtQGd3Z6rNfPmPV0VXc0vkWWge1vvgXCOECXKLw8koqOHQ6n56t5WRjW6w6uorZO2YzOGIwj/R7xOg4QjQYlzhoEX+m8naMMR3lcrI/orXmP0f+w4s/vsiQiCG8fvXr+Hr6Gh1LiAbjEoV3tqAMgPAmct3nhSTmJDJv9zx+yviJmMgYXh32Kj6ePkbHEqJBuUjhlQPQPFBGK+crqChgwf4FrExcSZBPEHMGzeGGjjfIKSjCLblI4ZUR6OtFgK9LrI7d5JXlMembSZwoOMHNnW5mep/pBPvKfk7hvlyiITIKy2geJKO76kpMJUzdOJVThad4Z9Q7DAwfaHQkIQznEts1afllNA+Um/X8KjEnkXu/vZdD2Yd4OeZlKTshqjh94RWWmTh0Op8eMv8dAMsOLeOW/95CSmEKLw99mRFRI4yOJITDcPpN2m2/ZGGyaEZ0kckqt6Zu5bV9rzGqzShmD5ot++uEOI/TF97GIxkE+3vTr02I0VEMdTzvOE9te4ouTbvwwlUvyPl1QtTAqQvPYtV8fySDYZ3D3HaG4xJTCYvjFrMiYQWNvBoxP2a+lJ0QF+DUhReXmkd2cQXD3XBzVmvN+pPrid0Ty9mSs1zf/noe6fcIzfybGR1NCIfl1IV3LLNySqhekU0MTtKwcspy+MeWf7A7fTddm3YlNiaW3s17Gx1LCIfn1IWXW1wBQNPG7nOJVH55PvdtuI8T+SeYOWAmN3e6GU8PmRJLCFs4deHllFTg5aEIdJMrLIpNxTzw3QMcyzvGW8Pf4sqIK42OJIRTceqmyCupICTAB6Vc/x4WpeZSpm6cSnx2PK8Oe1XKTohL4NSFl1NcQdNGrr85W2Gp4OHvH2Z/xn5euuolhkcNNzqSEE7JqQsvt9hESIC30THqlcVq4Z8//JMdZ3Ywd/BcxrYba3QkIZyWU5+8llNSQdMA1x3haa15bvdzfHvyW56IfkLuGStEHTl14WUXlRPiwpu0nx79lFVHV/G3y//Gnd3vNDqOEE7PaQuvtMJCbomJ8GDXnCUlKTeJl/e8zOBWg3mw74NGxxHCJTht4aX/Oq17sOtN615mLuOJrU8Q4B3A80Oel9mJhbATpz1ocTq3FMAlR3iv7nuVpLwkFo1cJJeKCWFHTjt0OHg6H4DOLQMNTmJfu9N289GRj7i96+1yrp0QdmZT4SmlxiqlEpVSSUqpGTU8/1elVFzVnx1KqV72j/pbP6Xk0q5ZAKGNXWdmkGJTMbO2z6JNUBvZbydEPbjoJq1SyhNYAIwCUoE9Sqm1WuuEaoudAGK01rlKqXHAYmBAfQSGytM19qfkMrST69yHttRcyuNbHietOI33x72Pv5fr7ZsUwmi27MPrDyRprY8DKKVWAhOAc4Wntd5RbfldQKQ9Q57vVE4pWUUV9I1yjUk/88vzmbZxGj9n/sz/Dfo/mflEiHpiS+FFAKeqPU7lj0dvfwPW1SXUxfyUkgvgEoWXWZLJ5A2TOVlwktiYWEa3HW10JCFcli2FV9OV+brGBZW6msrCG3KB5ycDkwGioqJsjPh7J7KKUQo6NG98ya/hCFILU7n323vJLstm4ciFcncxIeqZLQctUoHW1R5HAmfOX0gp1RNYAkzQWmfX9EJa68Va62itdXRY2KXvf0vLLyWssS8+Xk57kJnjece565u7KKgoYMnoJVJ2QjQAWxpjD9BRKdVOKeUD3Aqsrb6AUioK+Ay4Q2t91P4xfystv4zwJs67U/9IzhEmfTMJi9XCsjHL6BnW0+hIQriFi27Saq3NSqlpwHrAE1imtY5XSk2pen4RMAsIBRZWzU1n1lpH11fotPwyOoQ55+asyWLiiS1P4OPpw9IxS2kT1MboSEK4DZuutNBafw18fd7nFlX7+O/A3+0b7YJZSMsr5aqOznkFwoeHPyS5IJkFIxZI2QnRwJzu0rK8EhPFFRYinGiTttRcyqaUTXyR9AW70nYxNHIoQyOHGh1LCLfjdIWXnF15p7K2oQEGJ7HNvrP7eHTzo+SU5dAqoBX39bqP27vebnQsIdyS0xVeSk4JAG1CGxmc5I9prfn06Ke88OMLRDaO5JWhrxDdMlpmPhHCQE5XeMlZlYXXuqnjFl5RRRFzds5hffJ6rmx1JS8NfYlg32CjYwnh9pyu8E7mFBMe7Ieft2PeizU+O54ntjzBmaIzPNT3Ie65/B4Z1Qm7M5lMpKamUlZWZnQUw/j5+REZGYm3t+33tXG6wkvJLiHKQUd3a4+tZc6OOYT6h/Le2Pfo07yP0ZGEi0pNTSUwMJC2bdu6xW1Kz6e1Jjs7m9TUVNq1a2fz1znd0CM5u8ThDlhYtZU3f3qTmdtm0rd5Xz697lMpO1GvysrKCA0NdcuyA1BKERoaWusRrlON8ExWK1lF5UQ50AGLCksFM7fN5Jvkb7ix4408PfBpvD1c+9aRwjG4a9n96lLW36lGeMXlFsBxjtAWm4qZunEq3yR/w8N9H2bOoDlSdsJtKKW44447zj02m82EhYVx3XXX1ep12rZtS1ZWVp2XsYVTjfCKK8wADrEPL788nykbpnA45zDPXfkcEzpMMDqSEA0qICCAQ4cOUVpair+/Pxs2bCAiIsLoWH/IqUZ4VmvlrFSNfIw9Qmuymnh086MczT3K61e/LmUn3Na4ceP46quvAPjoo4+YOHHiuedycnL405/+RM+ePRk4cCBxcXEAZGdnM3r0aPr06cN9992H1v+bbe6DDz6gf//+9O7dm/vuuw+LxWLXvE41wnMUsXti+TH9R+YNmcew1sOMjiPc3DNfxpNwpsCur9mtVRCzx3e/6HK33norc+fO5brrriMuLo577rmHH374AYDZs2fTp08f1qxZw6ZNm7jzzjs5cOAAzzzzDEOGDGHWrFl89dVXLF68GIDDhw/z8ccfs337dry9vXnggQf48MMPufNO+92E3qkKr9xsBSDI37j9ZJ/98hn/OfIf7up2F+PbjzcshxCOoGfPniQnJ/PRRx9xzTXX/Oa5bdu2sXr1agCGDx9OdnY2+fn5bN26lc8++wyAa6+9lpCQypnLN27cyL59+7jiiisAKC0tpXnz5nbN61SFV1RmJtDXizCD7lS2P2M/z+56lsGtBvNwv4cNySDE+WwZidWn66+/nscff5zNmzeTnf2/uX+rb6r+6tcjqzUdYdVac9ddd/HCCy/UW1an2odXVG6mXViAIYfjf8n9hWkbpxHROIKXh76Ml4dT/a4Qot7cc889zJo1ix49evzm80OHDuXDDz8EYPPmzTRr1oygoKDffH7dunXk5lbeo2bEiBGsWrWKjIwMoHIf4MmTJ+2a1al+aovKzbSLaviTjlMLU7lvw334evqyaOQiuS5WiGoiIyN56KGHfvf5OXPmcPfdd9OzZ08aNWrEihUrgMp9exMnTqRv377ExMScu79Nt27deO655xg9ejRWqxVvb28WLFhAmzb2mzdS1TTsbAjR0dF67969Ni9vfb0na7JbkxLzGg+P7FSPyX5ry6ktzNoxC5PVxIqxK+gY0rHB3luICzl8+DBdu3Y1Oobhavp3UErtu9CM604zwrNYNRpoHdIw5+CVmEqI3RvLp0c/pVNIJ1666iU6hHRokPcWQtQPpyo8gOZB9X/AIqs0i7+t/xsn8k8wqfskpveZjo+nT72/rxCifjlN4f160nGzej5Cm1eWx73f3ktacRqLRi1icKvB9fp+QoiG4zSFZ9H1X3hFFUVM+W4KKQUpLBi5QO4VK4SLcZrTUixWjQKaBtTPpmVOWQ5TN04lMSeR+cPmS9kJ4YKcZoRn1RpfL088Pex7Dp7ZauaTxE94+8DblJpKeeGqF+RyMSFclBON8MDXy75xy8xl3PH1Hbzw4wt0C+3G6utXM7bdWLu+hxCuqnHjxjV+ftKkSaxateqSXnPOnDnExsbWJdYfcpoRntYaLy/7ju6+SPqCQ9mHmDt4Ln/q8Ce3n1BRCFfnNCM8DXbdnLVYLSyPX07PZj2l7ISoA60106ZNo1u3blx77bXnLg0D2LdvHzExMfTr148xY8aQlpYGwLvvvssVV1xBr169uOmmmygpKWmQrE4zwrNqjaeH/fp5Q8oGUotSeSz6MSk74dzWzYD0g/Z9zZY9YNyLNi36+eefk5iYyMGDBzl79izdunXjnnvuwWQyMX36dL744gvCwsL4+OOPmTlzJsuWLePGG2/k3nvvBeDpp59m6dKlTJ8+3b7rUAOnKTytwctOIzytNe8deo+2QW25uvXVdnlNIdzV1q1bmThxIp6enrRq1Yrhw4cDkJiYyKFDhxg1ahQAFouF8PBwAA4dOsTTTz9NXl4eRUVFjBkzpkGyOk3hAdhjIJZZksncXXNJyE5gzqA5eHo45v1thbCZjSOx+nSh6Z66d+/Ozp07f/fcpEmTWLNmDb169WL58uVs3ry5AVI60T68utJas/bYWiZ8MYGdZ3byePTj3NDxBqNjCeH0hg4dysqVK7FYLKSlpfH9998D0LlzZzIzM88VnslkIj4+HoDCwkLCw8MxmUznpopqCE41wquLV/e9yvL45fRp3oe5g+fSNrit0ZGEcAk33HADmzZtokePHnTq1ImYmBgAfHx8WLVqFQ8++CD5+fmYzWYefvhhunfvzrPPPsuAAQNo06YNPXr0oLCwsEGyOs30UGee6cTpoF5c8cintX6vxJxE/vLfvzCh/QRmD5otm7HC6cn0UJVqOz2Uy2/Saq154ccXCPIJ4rHox6TshHBjLl9465PXs+/sPh7s+6DMVCyEm3PZfXil5lKO5Bwhdm8sXZt25cYONxodSQhhMJcrvGJTMY9tfoxdabuwaAs+Hj7ExsTKpqwQwrUKz2K18OTWJ9mVtos7u99Jn7A+9AzrSah/qNHRhBAOwGUKT2tN7N5YtqRu4akBTzGxy0SjIwkhHIxLHLQoNZfyxNYn+ODwB/y161+l7IRwQm3btiUrK6te38PpR3jpxek8uOlBjuQc4dF+jzKp+ySjIwnhdrTWaK3xsOMEH/XBqQsvITuBqRunUmou5a3hbxHTOsboSEK4jeTkZMaNG8fVV1/Nzp076d27NwcPHqS0tJQ///nPPPPMM0DlyO2uu+7iyy+/xGQy8emnn9KlSxeys7OZOHEimZmZ9O/fn4a4CMKmwlNKjQXeADyBJVrrF897XlU9fw1QAkzSWv9k56y/sTd9L9M2TSPYJ5h3x70r94wVbuulH1/iSM4Ru75ml6ZdeLL/kxddLjExkffee4+FCxeSk5ND06ZNsVgsjBgxgri4OHr27AlAs2bN+Omnn1i4cCGxsbEsWbKEZ555hiFDhjBr1iy++uorFi9ebNd1qMlFx59KKU9gATAO6AZMVEp1O2+xcUDHqj+TgX/ZOedvbE3dypTvptCiUQtWjFshZSeEQdq0acPAgZU3vPrkk0/o27cvffr0IT4+noSEhHPL3Xhj5Xmw/fr1Izk5GaicVur2228H4NprryUkJKTe89oywusPJGmtjwMopVYCE4CEastMAN7XlWPSXUqpJkqpcK11mj1Caq056KvY6ZfN+xuns+30Njo37cy/Rv6LEL/6/0cSwpHZMhKrLwEBAQCcOHGC2NhY9uzZQ0hICJMmTaKsrOzccr6+lbdX9fT0xGw2n/t8Q0++a8sexgjgVLXHqVWfq+0yKKUmK6X2KqX2ZmZm2hxSKcXcsABWN8okuSCZmzrdxJLRS6TshHAQBQUFBAQEEBwczNmzZ1m3bt1Fv2bo0KHnpoZat24dubm59R3TphFeTRV8/t5FW5ZBa70YWAyVs6XY8N7nPH3VYloHtuLy8Fa1+TIhRAPo1asXffr0oXv37lx22WVceeWVF/2a2bNnM3HiRPr27UtMTAxRUVH1nvOi00MppQYBc7TWY6oe/xNAa/1CtWXeATZrrT+qepwIDPujTdraTg8lhPgfmR6qUn1MD7UH6KiUaqeU8gFuBdaet8xa4E5VaSCQb6/9d0IIYS8X3aTVWpuVUtOA9VSelrJMax2vlJpS9fwi4GsqT0lJovK0lLvrL7IQQlwam87D01p/TWWpVf/comofa2CqfaMJIYR9OfZ1IEKICzLq9gyO4lLWXwpPCCfk5+dHdna225ae1prs7Gz8/Pxq9XVOfS2tEO4qMjKS1NRUanM+q6vx8/MjMjKyVl8jhSeEE/L29qZdu3ZGx3A6skkrhHAbUnhCCLchhSeEcBsXvbSs3t5YqUzgZC2/rBlQv3NANxxXWRdXWQ+QdXFUtV2XNlrrsJqeMKzwLoVSau+FrpFzNq6yLq6yHiDr4qjsuS6ySSuEcBtSeEIIt+FshVf/k943HFdZF1dZD5B1cVR2Wxen2ocnhBB14WwjPCGEuGQOWXhKqbFKqUSlVJJSakYNzyul1JtVz8cppfoakfNibFiPv1blj1NK7VBK9TIipy0uti7VlrtCKWVRSv25IfPVhi3ropQappQ6oJSKV0ptaeiMtrDh+ytYKfWlUurnqvVw2HkqlVLLlFIZSqlDF3jePj/zv94x3FH+UDnJ6DHgMsAH+Bnodt4y1wDrqLyXxkBgt9G5L3E9BgMhVR+Pc8T1sHVdqi23icq5E/9sdO46/L80ofKufFFVj5sbnfsS1+Mp4KWqj8OAHMDH6OwXWJ+hQF/g0AWet8vPvCOO8M7dFlJrXQH8elvI6s7dFlJrvQtoopQKb+igF3HR9dBa79Ba/3qrpl1A7aZ+aDi2/J8ATAdWAxkNGa6WbFmX24DPtNYpAFprR1wfW9ZDA4Gq8l6IjaksPDMOSGu9lcp8F2KXn3lHLDy73RbSYLXN+Dcqf4M5oouui1IqArgBWIRjs+X/pRMQopTarJTap5S6s8HS2c6W9Xgb6AqcAQ4CD2mtrQ0Tz+7s8jPviNND2e22kAazOaNS6moqC29IvSa6dLasy+vAk1prS0PfXLmWbFkXL6AfMALwB3YqpXZprY/Wd7hasGU9xgAHgOFAe2CDUuoHrXVBfYerB3b5mXfEwksFWld7HEnlb6jaLmM0mzIqpXoCS4BxWuvsBspWW7asSzSwsqrsmgHXKKXMWus1DRPRZrZ+f2VprYuBYqXUVqAX4EiFZ8t63A28qCt3giUppU4AXYAfGyaiXdnnZ97onZU17Jz0Ao4D7fjfztju5y1zLb/dgfmj0bkvcT2iqLzT22Cj89Z1Xc5bfjmOe9DClv+XrsDGqmUbAYeAy43Ofgnr8S8q7ykN0AI4DTQzOvsfrFNbLnzQwi4/8w43wtMucltIG9djFhAKLKwaGZm1A17wbeO6OAVb1kVrfVgp9Q0QB1iBJVrrGk+XMIqN/yfPAsuVUgepLIontdYOOYOKUuojYBjQTCmVCswGvMG+P/NypYUQwm044lFaIYSoF1J4Qgi3IYUnhHAbUnhCCLchhSeEcBtSeEIItyGFJ4RwG1J4Qgi38f/0kshdlv6yJwAAAABJRU5ErkJggg==\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(5,5)) # 使图表为正方形\n",
    "plt.plot(df_scores.fpr,df_scores.tpr,label='Model')\n",
    "plt.plot(df_idea.fpr,df_idea.tpr,label='Ideal')\n",
    "plt.plot(df_rand.fpr,df_rand.tpr,label='rand')\n",
    "plt.legend()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T02:04:50.820730Z",
     "start_time": "2023-12-09T02:04:50.570738100Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.86147291e+00 8.61472905e-01 7.76983067e-01 7.76290758e-01\n",
      " 7.68914300e-01 7.68870095e-01 7.67692910e-01 7.66966266e-01\n",
      " 7.66636305e-01 7.66492419e-01 7.56364822e-01 7.55803281e-01\n",
      " 7.55384262e-01 7.54817436e-01 7.54517017e-01 7.54081271e-01\n",
      " 7.40639245e-01 7.38554561e-01 7.35984044e-01 7.34714768e-01\n",
      " 7.33715048e-01 7.33286224e-01 7.32551737e-01 7.32344661e-01\n",
      " 7.31760914e-01 7.30482237e-01 7.29757711e-01 7.29034039e-01\n",
      " 7.27344138e-01 7.22349435e-01 7.21101720e-01 7.20841059e-01\n",
      " 7.18767940e-01 7.18636349e-01 7.18529990e-01 7.17011034e-01\n",
      " 7.12799754e-01 7.12743227e-01 7.12506653e-01 7.09898497e-01\n",
      " 7.00785955e-01 7.00655513e-01 6.93282892e-01 6.92885201e-01\n",
      " 6.92279275e-01 6.91263352e-01 6.89833597e-01 6.88886575e-01\n",
      " 6.87747757e-01 6.86655686e-01 6.85813590e-01 6.82239090e-01\n",
      " 6.81797355e-01 6.81007413e-01 6.75666335e-01 6.74824598e-01\n",
      " 6.74358303e-01 6.74021823e-01 6.71591154e-01 6.65694174e-01\n",
      " 6.62916242e-01 6.58730903e-01 6.57284168e-01 6.57263204e-01\n",
      " 6.56617207e-01 6.56070334e-01 6.49289647e-01 6.48770369e-01\n",
      " 6.47155145e-01 6.46439246e-01 6.46143311e-01 6.45667986e-01\n",
      " 6.45411166e-01 6.44865523e-01 6.43170469e-01 6.41474439e-01\n",
      " 6.35953650e-01 6.32610660e-01 6.31528092e-01 6.30739279e-01\n",
      " 6.30457913e-01 6.30112019e-01 6.29929547e-01 6.28837586e-01\n",
      " 6.28227843e-01 6.27406832e-01 6.26377776e-01 6.24980074e-01\n",
      " 6.22848956e-01 6.21757218e-01 6.19973466e-01 6.19967633e-01\n",
      " 6.18501111e-01 6.17419459e-01 6.14131471e-01 6.13854128e-01\n",
      " 6.12785946e-01 6.12210794e-01 6.12125844e-01 6.11920336e-01\n",
      " 6.11349439e-01 6.10443839e-01 6.07475617e-01 6.07249057e-01\n",
      " 6.03554409e-01 6.01994049e-01 6.01894784e-01 6.01859745e-01\n",
      " 6.01264503e-01 6.00346870e-01 5.98717001e-01 5.96661650e-01\n",
      " 5.93793624e-01 5.92248437e-01 5.91939720e-01 5.91299640e-01\n",
      " 5.85842908e-01 5.85296454e-01 5.84944703e-01 5.84483831e-01\n",
      " 5.82208589e-01 5.81993378e-01 5.78804307e-01 5.77855054e-01\n",
      " 5.74602811e-01 5.73749084e-01 5.73667123e-01 5.70739330e-01\n",
      " 5.70164955e-01 5.67282006e-01 5.67275581e-01 5.67069824e-01\n",
      " 5.66818574e-01 5.63369843e-01 5.63164192e-01 5.62944287e-01\n",
      " 5.62759055e-01 5.62061819e-01 5.61818684e-01 5.57547878e-01\n",
      " 5.45831780e-01 5.44994599e-01 5.42471728e-01 5.41929121e-01\n",
      " 5.39785062e-01 5.37695607e-01 5.36814261e-01 5.36219864e-01\n",
      " 5.35525747e-01 5.35108551e-01 5.33452145e-01 5.33410498e-01\n",
      " 5.32156904e-01 5.31471295e-01 5.27224430e-01 5.26253611e-01\n",
      " 5.25288480e-01 5.24859494e-01 5.22774292e-01 5.20877498e-01\n",
      " 5.17881953e-01 5.17667227e-01 5.14338125e-01 5.14016052e-01\n",
      " 5.08809291e-01 5.08034744e-01 5.07419722e-01 5.05571482e-01\n",
      " 5.03154329e-01 5.01702714e-01 5.00692854e-01 5.00215024e-01\n",
      " 4.97484271e-01 4.97316425e-01 4.94091761e-01 4.93771180e-01\n",
      " 4.91939985e-01 4.91936936e-01 4.91230811e-01 4.89572353e-01\n",
      " 4.87881372e-01 4.86317141e-01 4.85495805e-01 4.85324394e-01\n",
      " 4.84909064e-01 4.83689577e-01 4.82865021e-01 4.82837326e-01\n",
      " 4.79680565e-01 4.78950581e-01 4.75209745e-01 4.74694419e-01\n",
      " 4.74532233e-01 4.72799539e-01 4.72519003e-01 4.71156170e-01\n",
      " 4.70670005e-01 4.69219471e-01 4.67562438e-01 4.66257633e-01\n",
      " 4.61826943e-01 4.61376933e-01 4.59553457e-01 4.59234500e-01\n",
      " 4.57653607e-01 4.56060599e-01 4.55864320e-01 4.54758668e-01\n",
      " 4.51891136e-01 4.51284948e-01 4.50780173e-01 4.49124595e-01\n",
      " 4.44774195e-01 4.42781934e-01 4.42658231e-01 4.41274555e-01\n",
      " 4.40985978e-01 4.39849732e-01 4.39441701e-01 4.38992953e-01\n",
      " 4.38047237e-01 4.36415972e-01 4.34802379e-01 4.33870577e-01\n",
      " 4.26896204e-01 4.26727084e-01 4.25960517e-01 4.24246296e-01\n",
      " 4.15595820e-01 4.15067306e-01 4.13191212e-01 4.09223888e-01\n",
      " 4.09193354e-01 4.08440861e-01 4.07722183e-01 4.07387549e-01\n",
      " 4.04547029e-01 4.03016402e-01 4.00213633e-01 3.99795416e-01\n",
      " 3.97046575e-01 3.95578379e-01 3.88523945e-01 3.84167402e-01\n",
      " 3.82591190e-01 3.82560849e-01 3.78573122e-01 3.78499375e-01\n",
      " 3.78450474e-01 3.77616582e-01 3.76454164e-01 3.74859991e-01\n",
      " 3.74200917e-01 3.72769393e-01 3.71383480e-01 3.70669572e-01\n",
      " 3.70073037e-01 3.68076377e-01 3.65668737e-01 3.65082559e-01\n",
      " 3.64642872e-01 3.61935761e-01 3.57593698e-01 3.53924359e-01\n",
      " 3.50854235e-01 3.50747395e-01 3.48981895e-01 3.48600425e-01\n",
      " 3.46859194e-01 3.46794284e-01 3.44212833e-01 3.40195223e-01\n",
      " 3.36699489e-01 3.35773949e-01 3.35258148e-01 3.35084417e-01\n",
      " 3.25859403e-01 3.21781425e-01 3.21000473e-01 3.19499152e-01\n",
      " 3.18703758e-01 3.18533521e-01 3.18527319e-01 3.17891954e-01\n",
      " 3.15900083e-01 3.14517598e-01 3.11092031e-01 3.09052258e-01\n",
      " 3.08870645e-01 3.07833020e-01 3.06256357e-01 3.03986325e-01\n",
      " 3.03575548e-01 3.02686040e-01 3.01406196e-01 2.99538735e-01\n",
      " 2.99022778e-01 2.98679524e-01 2.98177914e-01 2.97853174e-01\n",
      " 2.97340851e-01 2.96513168e-01 2.95377566e-01 2.94930091e-01\n",
      " 2.88702070e-01 2.87835927e-01 2.85073770e-01 2.84067886e-01\n",
      " 2.82040698e-01 2.81419077e-01 2.80002545e-01 2.77284836e-01\n",
      " 2.75794826e-01 2.75517904e-01 2.71831698e-01 2.71235559e-01\n",
      " 2.70024127e-01 2.70018738e-01 2.64838816e-01 2.64717392e-01\n",
      " 2.64251188e-01 2.63962142e-01 2.61384174e-01 2.61297256e-01\n",
      " 2.56197965e-01 2.54843198e-01 2.52439369e-01 2.52028561e-01\n",
      " 2.50010203e-01 2.49878167e-01 2.48072859e-01 2.47045541e-01\n",
      " 2.40661365e-01 2.40609812e-01 2.34146968e-01 2.33418780e-01\n",
      " 2.31710209e-01 2.31387204e-01 2.27501029e-01 2.26257822e-01\n",
      " 2.23237275e-01 2.22558809e-01 2.21694144e-01 2.20934568e-01\n",
      " 2.19780194e-01 2.19367244e-01 2.12016526e-01 2.11907883e-01\n",
      " 2.09534272e-01 2.09506553e-01 2.06695059e-01 2.05589607e-01\n",
      " 1.99962326e-01 1.99900548e-01 1.96794077e-01 1.95720603e-01\n",
      " 1.94192399e-01 1.93867055e-01 1.93833174e-01 1.93765426e-01\n",
      " 1.93731558e-01 1.93562289e-01 1.93117625e-01 1.92566846e-01\n",
      " 1.91826309e-01 1.91080976e-01 1.90423601e-01 1.89070326e-01\n",
      " 1.87704889e-01 1.87626951e-01 1.86650667e-01 1.86349809e-01\n",
      " 1.84196058e-01 1.83741358e-01 1.83472769e-01 1.82981242e-01\n",
      " 1.82207886e-01 1.82058141e-01 1.81854110e-01 1.81760818e-01\n",
      " 1.81582457e-01 1.80060744e-01 1.74566076e-01 1.73826299e-01\n",
      " 1.71603493e-01 1.70703335e-01 1.69562617e-01 1.69164349e-01\n",
      " 1.67019423e-01 1.66844565e-01 1.66790615e-01 1.66729682e-01\n",
      " 1.66609913e-01 1.65687713e-01 1.65065807e-01 1.64591053e-01\n",
      " 1.59710780e-01 1.59472483e-01 1.59210380e-01 1.58506782e-01\n",
      " 1.55408548e-01 1.55318515e-01 1.54097650e-01 1.52706013e-01\n",
      " 1.49606991e-01 1.49080886e-01 1.47172089e-01 1.46308924e-01\n",
      " 1.45868679e-01 1.45776251e-01 1.44763605e-01 1.43524555e-01\n",
      " 1.38008847e-01 1.37647300e-01 1.35884990e-01 1.35809653e-01\n",
      " 1.32562400e-01 1.32499731e-01 1.28056218e-01 1.27952185e-01\n",
      " 1.18634890e-01 1.18341202e-01 1.17027077e-01 1.16375673e-01\n",
      " 1.10079104e-01 1.10053134e-01 9.47492715e-02 9.25494844e-02\n",
      " 8.93770603e-02 8.93167250e-02 8.09944988e-02 8.09180352e-02\n",
      " 7.76880223e-02 7.76181018e-02 7.55261887e-02 7.53828831e-02\n",
      " 7.14466277e-02 7.09230769e-02 6.98449311e-02 6.96823304e-02\n",
      " 6.28540994e-02 6.27953540e-02 5.52897691e-02 5.47733430e-02\n",
      " 5.40899234e-02 5.40765945e-02 4.44765633e-02 4.44266226e-02\n",
      " 4.08979165e-02 4.08725043e-02 2.62818654e-02 2.62736243e-02\n",
      " 1.39371127e-02 1.39175264e-02 1.36503487e-02 1.34993592e-02\n",
      " 1.33503561e-02 1.32024167e-02 9.11171323e-03 9.08028490e-03\n",
      " 4.24260521e-03 4.21368098e-03 1.75012676e-03]\n"
     ]
    },
    {
     "data": {
      "text/plain": "<Figure size 360x360 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATwAAAEvCAYAAADYR30zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xV9f3H8dc3mxAggQBC2Fv2iIg4EVR22LIDVamz1l/raK2jrbVWba3WQZFSlhBFkBmctYCVjWwE2YQZSAgJ2bnf3x8nYAyB3CT35nvPPZ/n4+HD3NzDve9Dct+c+f0qrTVCCOEEAaYDCCFEZZHCE0I4hhSeEMIxpPCEEI4hhSeEcAwpPCGEYwSZeuPo6GjdpEkTU28vhPBTmzdvPqu1rl3Sc8YKr0mTJmzatMnU2wsh/JRS6sjVnpNdWiGEY0jhCSEcQwpPCOEYUnhCCMeQwhNCOIYUnhDCMaTwhBCOUWrhKaVmKKXOKKV2XuV5pZR6Sym1Xym1XSnV1fMxhRCi4tzZwpsJ9L3G8/2AloX/TQHeq3gsIYTwvFLvtNBar1ZKNbnGInHAbG0NnbxOKRWplKqntT7poYxCCD+SnJ7DV3tOU9pY68qVR5tDc6h79y+pVyvSI+/tiVvLYoBjRR4nFX7visJTSk3B2gqkUaNGHnhrIYSnFLg0W4+lkpPnKnXZk2nZvPHlPsJDAlGoMr3P3tPppS4TQh5vB79F58DN7Krbinp3ji3Te1yNJwqvpLUtsby11tOAaQCxsbEymYYQPkJrzYC31vD9qdLLqKhmtavSqk5Emf5M0+iqxERV4YFbm5W8QH42kcvvI+zQZi70+jPNe95bpte/Fk8UXhLQsMjjBsAJD7yuEOIqzmXk8K9vDpFXUPrWmDsWbTnOuYu5AMz6WXfCgko/vB8RFkTbetVRqmxbeNeUmwkJ8XBoFQx6k+rdJnnutfFM4S0FHlVKJQA3Amly/E6In9qedJ5tSWnWA615bskuAIICylcW+a4fd5DCQwIrnO9ScW58tg+1q4VW+PXKJScD5o+Gw99A3DvQZZzH36LUwlNKzQfuAKKVUknAC0AwgNZ6KpAI9Af2A5nAZI+nFMJHPfHhVrYdO09pGzkHki9e8b0GUVWI61y/3O8dHRHKpJ5NPLuFZUr2BfhgJCRthGHvQ8eRXnkbd87SjinleQ084rFEQhhwMSefcxm5V3xfo3n9831czMm/4rlTadnsPnkBgAEd613z9dtcV51ebepweytrXMrAAEXNqiEeSO4Hss7D3OFwciuMmAHthnjtrYwNACqEr5i+5iAvrdhT6nIdYmr85HFAALSsE8FfR3WiYwPPXDbhOJkpMGcInN4No2ZDmwFefTspPOEIe05eYP3Bcxw6e5FZa60BcZUCXeRagZ7NazG8a4Mr/mxQoOLONnWoFhZcWXGdISPZKruzP8DoedDqbq+/pRSesKWTaVmlXkKx+LvjbDmaSqBSHD6X+ZPnBnWqT9Na4QAEBQYwpnsjcwfrnSj9NMweDKlHYGwCNL+zUt5WCk/4hPwCF0dTMq+5jEvDKyv3kFegWbUv2e3Xjutcn04NI7mpWS3uaXcdocEBhIfIr74xF07ArEFw4SSMWwBNb620t5afujBCa82iLcfZlnQegNlrrzrvSok6NahB+5gajOh25S5oUQ1rhhMdIVtuPuP8MavsLp6FCYugUY9KfXspPOF1eQUuFm1JIiOnAIACl4uXE7+//HxUeDDVwoKoHhbMU31bX/O1ggMD6NW6DlU8cO2ZqGQph2DWYMhOg4mLoUFspUeQwhNeselwCvfP3kTVkCCOn88qcZm29arzj7FdaF67bLcmCRs6d8DassvLhPglUL+LkRhSeMLjlmw9zuMJWwEIUIphXWMIDgjg8T4tqRpq/coFBigiQuXXzxGS91pbdq48iF8G13UwFkV+44RHpFzM5a2vfiC3wMW3+88C8LdRnRjaJcY/7gQQ5XN6t3U2FgWTVkCd643GkcITFXYhO4+uf/zi8uPoiFBGdmvAsBKuaRMOcnIbzB4CQaHWll10S9OJpPBE+W04lML0NQf5fPdpAFrVjeDTx28joJw3xAs/cnwzzBkKIdUgfinUam46ESCFJ8rpj8t3869vDl1+PKlnE57p10bKTsCxDda9sVWirC27qMamE10mhSfK7GRa1uWye29cV3o2j6ZGuNx2JYDD/4N5oyCirrVlV8O3DmtI4Qm3HT2XyYodJ/nhjHVL12N3tqBfh2uPEiIc5OB/Yf4Yq+Til0G160wnuoIUnnDb5JkbLo/rViU4kNHdZV4SUWj/l5AwDmo2g4lLIKKO6UQlksITbjuQfJGgAMXO399DYIAiOFDmcRfA3pXw0USo3RomLIGqtUwnuiopPHFN2XkFvPHFPvYUjkzSvHYEYcFyW5cotHspfDzZuph4/CIIr2k60TVJ4YkSbT12noQNR0nY+OMMnLWqhvDysPYGUwmfsnMhLHwAYrrB+I8hrEbpf8YwKTzxE2mZefT481dk5RVc/t6Y7o14um9rIsNlSHJRaFsCLH4IGvaAcR9BaDXTidwihScuS8vKo9+bq8nKKyAqPJjXRnSiV5s6BMq1daKoLXNg6WPWOHZjEiCkqulEbpPCEwCkZ+fR6fefX3685bm75B5YcaWN/4IV/wfNe8PoDyC4iulEZSKn2QS7TqTR4UWr7GpWDWHNU72k7MSV1k21yq5VX2sOCpuVHcgWngAGvPUNAMO7NuBPQ9vLWVhxpf+9CV88D9cPguEzIMiex3Ol8BzufOaPc7H+dVQng0mEz1r1Gnz9ErQbBsOmQaB9byOUwnMwrTXPLdkFwHMD2xpOI3yO1vD1y7D6Veg4GuLegUB7V4a904sKWbrtBMu2nQAgtnGU4TTCp2gNX74I//s7dBkPg96CAPsf6pDCcyit9eVh2Nc81YuGNcMNJxI+Q2v47Lew7l2IvQ/6vw4B/nF+UwrPoZ740Cq7ZrWrStmJH7lcsPJJ2DgdbnwI+v4Z/OiMvRSeA2mtWbzV2pWdd3/lzgsqfJjLBcsfhy2zoecv4K4/+FXZgRSeo+w6kcby7SdZsMm6P/bRXi24rkaY4VTCJ7gKYMkjsG0+3PYk9HrW78oOpPAc4UByBuOnr+dkWvZPvh/fs4mZQMK3FOTDJz+HnR9bRXf7U6YTeY0Unp+6mJPPQx9s4XRaNntPp1/+/j8ndOOedr43Eq0wJD8XFt4He5ZCn9/DLb80ncirpPD8kNaadi98dvnxPe3q0rVRFPff2kwGAhA/ys+Bj+Jh30q4589w08OmE3mdFJ4fSUrN5IP1R5mz9sjl7+148W6qhdn3ynjhJXlZ8OF4a2j2/q9D9wdMJ6oUUnh+YtW+ZOJnbLj8OCw4gLXP9JayE1fKzYT5o+HQauuC4m7xphNVGik8m9p/JoNTadnMXXeEfWfSuZCVB8Bv+rXh57f7xqTHwgflZMC8e+HotzDkPeg8xnSiSiWFZ0NbjqYy7N1vf/K9AR3qUT8yjCm3NTOUSvi87DT4YCQkbYJh70OHEaYTVTopPJtZd/Aco6etA2DKbc24q21dWtSOIKqqPYfrEZUkKxXmDINT22Hkv6FtnOlERkjh2URegYsH52zmq+/PAPBU39Y8fEcLw6mELVw8B3OGQPL3MGoOtOlvOpExUng2kJ6dxy8Ttl4uu7fHdmFgx/qGUwlbyEiG2XFwbj+Mng8t+5hOZJRbhaeU6gu8CQQC07XWrxR7vgYwF2hU+Jqva63/7eGsjpSTX3B5+HWADb/tTZ3qcjuYcEP6KZg1GM4ftWYWa3aH6UTGlTrmi1IqEHgH6Ae0BcYopYqPFvkIsFtr3Qm4A/irUkoOKlWQy6Vp89ynlx+veaqXlJ1wT9px+Hd/SEuy5oxtdofpRD7BnUGuugP7tdYHtda5QAJQ/IinBqopa+aXCCAFyPdoUofJzXfR6fefo7X1eNvzd8swTsI954/CzP6QcQYmLIImt5hO5DPc2aWNAY4VeZwE3FhsmbeBpcAJoBpwr9ba5ZGEDrXpcArpOda/GdteuJsaVeQCYuGGlEMwaxDkXICJS6BBN9OJfIo7W3gl3Xypiz2+B9gK1Ac6A28rpapf8UJKTVFKbVJKbUpOTi5zWCc5lpoJwIIHb5KyE+45u9/ajc3NgIlLpexK4E7hJQENizxugLUlV9RkYJG27AcOAW2Kv5DWeprWOlZrHVu7du3yZvZ7v/1kB08v3AFAFZkyUbjjzPfWbmxBLsQvh/qdTSfySe4U3kagpVKqaeGJiNFYu69FHQV6Ayil6gKtgYOeDOoUaw+cY976owD8Ma4d7epfsaEsxE+d2gkzB1hfT1oB17U3m8eHlXoMT2udr5R6FPgM67KUGVrrXUqpBwufnwr8EZiplNqBtQv8tNb6rBdz+61pqw8AsPChnnSTmcREaU5stS4qDqoC8csgWi5Gvxa3rsPTWicCicW+N7XI1yeAuz0bzXlcLs3Xe61jm10aRhpOI3xe0maYOxRCq0P8Uqgp91GXxj/mXvMT76+xjgJcX686ATJQp7iWo+utOyiqRMHkRCk7N0nh+ZBNR1IBSJgiM4mJazj8DcwZChF1YFIiRDYyncg2pPB8xMbDKXyx+zSAXIYiru7A1zB3BNRoYG3Z1YgxnchWpPB8QG6+i0c+2ALAM/2uuJpHCMsPX1qDd9ZsZp2NrSaTMZWVFJ4P+MPyXZxJzyEmsgoPymjFoiR7V0LCGKjdGiYthwi5jrU8pPAM23PyAnPXWdfdzXug+B17QgC7l1gT7tRtb52NDa9pOpFtSeEZ1u/NNQD8sk9LGteqajiN8Dk7PoYFkyGmG0xcbJ2VFeUmhWfQ2gPnLn/9eO+WBpMIn7R1Hix6ABr1gPELIayG6US2J4VnyOGzFxnzvjU3xfsTY7FG1hKi0OZZsPhhaHIrjFsAodVMJ/ILUniGTCu8yHhktwb0ai0HoEURG96HZb+AFr1h7IcQIoc6PEUKz4ACl748QMCT97QmKFB+DKLQ2nch8dfQqh+MngfBVUwn8ivySTPgd4t3AjCgYz0Zsl386Js34LPfwPWDYdRsCAo1ncjvyKxllWzT4RTmb7C27l6Kk2F8RKFVr8LXf4L2w2HoNAiUj6Y3yN9qJVu4JQmAx+5sIZNnC9DaKrrVr0GnMRD3DgTIoK/eIoVXic5n5jJ/wzGiwoP51d2tTccRpmkNXzwP374FXSfCwDchQI4yeZMUXiU6mZYNwE3NaxlOIozTGj79Dax/D264H/q9JmVXCaTwKkmBSzNu+noA+neoZziNMMrlgsRfwaYZ0ONhuOdlkOswK4UUXiXZdzqdlIu5APRqXcdwGmGMq8C6xu67uXDzL6HPi1J2lUgKr5JMnLEBgLfGdKFqqPy1O1JBPix5BLYnwO1Pwx2/kbKrZPLJqwQn07JITs8BYIDszjpTQR4smgK7FkGv38HtT5pO5EhSeF6WkZPPiPfWAvBU39YEylwVzpOfCx9Phu+Xw11/gJsfN53IsaTwvGzNvmSOn88iKEAx+gaZe8Bx8rJhQTzs+xT6vgI9HjKdyNGk8Lxk2bYTPDb/u8uPFz9yMzXlQmNnycuChHFw4CsY8De44T7TiRxPCs8LTqZl8Y///ADAsK4xdGoQSfsYGcvMUXIvwvzRcGgNDH4buk4wnUgghedx+8+k0+dvqwHo0iiSv43qbDiRqHQ56fDBKDi2Dob+EzrdazqRKCSF50F5BS7Gvm9dXBzXuT7PD2xrOJGodNlp1jSKxzfD8OnWYADCZ0jhedDeU+mcKbz85PWRnQiWce6cJSvVmiD71E4YORPaDjadSBQjhedBWlv/nz4xVsrOaS6egzlxkLwX7p0LrfuaTiRKIIXnQS8u22U6gjAh4wzMjoOUgzBmPrToYzqRuAopPA+Zs+4Im4+kAnB9/eqG04hKc+EkzB4M549Z8080u8N0InENUngecuTsRQDWPNWLmEiZh8AR0pJg1iBrC2/8Qmhys+lEohRSeBWUlJrJD6czmP7NIQIUNKwZbjqSqAypR6yyy0qFCZ9Aw+6mEwk3SOFVgNaaW/7y9eXHDaKk7Bwh5SDMGgw5F2DiYojpZjqRcJMUXgV8uvMUANERIfwr/gauryfH7vze2R+sLbv8HIhfBvU6mU4kykAKr5y01jz0wRYAXh3RkU4NIw0nEl53Zo+1ZYeGScuhbjvTiUQZycVi5TSvcKrF2MZR3NmmruE0wutO7YCZA0AFwKQVUnY2JYVXTs9+Yk2m/fKwDoaTCK878R3MHAhBYTA5EWrLjHN2JYVXDqmFc1OEBgXQqm41w2mEVyVtgllxEFrdKrtazU0nEhUghVcOf1i+G4Bn+rUxnER41ZG1MHsIhEfB5BUQ1cR0IlFBUnhltD3pPJ98dxyA4d0aGE4jvObQGpg7HKrVhckrIVJGq/YHbhWeUqqvUmqvUmq/UuqZqyxzh1Jqq1Jql1JqlWdj+oZ9p9MZ/Pb/ALihSRTVw4INJxJeceA/8MFIiGwIkxKhen3TiYSHlHpZilIqEHgHuAtIAjYqpZZqrXcXWSYSeBfoq7U+qpTyy4lX737DGtgz/qbG/E7GuvNP+z6HD8dDdEuYuASqRptOJDzInS287sB+rfVBrXUukADEFVtmLLBIa30UQGt9xrMxzdJa8+FG6zKUzg0jeWFQOxn+yR99vwISxkKdNtZFxVJ2fsedC49jgGNFHicBNxZbphUQrJT6L1ANeFNrPdsjCX3ALX/5muPnswAY36MxATLVov/Z9QksvN+6c2L8IqgiF5L7I3cKr6RPty7hdboBvYEqwFql1Dqt9b6fvJBSU4ApAI0a2eMg8KrCaRYBvn3mTurLSCj+Z/sC+GQKNOgO4xZAmNwi6K/c2S9LAhoWedwAOFHCMp9qrS9qrc8Cq4ErbjLUWk/TWsdqrWNr165d3syVakfSeQCWP3aLlJ0/2joPFj0AjW+2hniSsvNr7hTeRqClUqqpUioEGA0sLbbMEuBWpVSQUioca5d3j2ejmrFqXzIAra+TC4z9zuaZsPhhaHY7jP0IQiNMJxJeVuourdY6Xyn1KPAZEAjM0FrvUko9WPj8VK31HqXUp8B2wAVM11rv9GbwynAgOYONh61RjOUkhZ/Z8D4k/hpa3GXNQREcZjqRqARujZaitU4EEot9b2qxx68Br3kumlkpF3Pp/VfrcsIujeQAtl/59m34/FloPQBG/huCQk0nEpVENluuYui71gXGvdvUYcHPbzKcRnjMmr9ZZdc2DkbNkrJzGBkPrwRHz2Vy5FwmANMmxhIol6HYn9aw6lX478vQYSQMmQqB8uvvNPITL8HmoykAPNGnlZSdP9Aa/vNHWPNX6DQW4t6GgEDTqYQBUnglyMwtAGBAx3qGk4gK0xo+/x2sfRu6xsPAv0OAHMlxKim8Elwa3DM8RLYCbE1rWPk0bPgn3PAA9HtVys7hpPCKuTQxDyAXGtuZywUr/g82/xtuehTufgmUHJ5wOim8Yj7ebN02vOQRmVTZtlwFsPQXsHUu3PIE9H5Byk4AUng/sfi743y5xxropWODGobTiHIpyIfFD8GOj+D2Z+COZ6TsxGVSeEU8t8Q6dvfeuK4o+ZDYT0GeNeLJ7sVw53Nw269NJxI+Rgqv0LTVB0jPzqdqSCD9OsjZWdvJz4GPfwbfL7eO1/V8zHQi4YOk8ApNX3MIgMVy7M5+8rLho4nww2fWmdgbf246kfBRUnjA7xbv4Ex6DlHhwbSUaRftJTcTPhxnzUMx8A2I/ZnpRMKHOb7wzmXkMHedNXz7y0NlUm1byb0I8+6Fw99A3DvQZbzpRMLHOb7w4t6xBgl4cVBbOXZnJ9kXYN4oOLYehv4TOt1rOpGwAUcX3ortJ0lKtYZvv/cGeww5L4Cs8/DBCDi+BYb/C9oPM51I2IRjC09rzTMLtwOw4MGbqCK3kdlDZgrMGQqnd1nDO10/yHQiYSOOLbzvT6WTnpPPjU1rckOTmqbjCHdcPAuzh8DZvdYoxa37mk4kbMaxhZeZmw/A5JubmA0i3JN+GmbHQeohGJMALXqbTiRsyLGFdyHbKrzwEMf+FdjHhRMwazBcOG5NttPsdtOJhE059tP+3VFr+kUZAsrHnT8GswbBxWRrGsXGPU0nEjbm2MLbdyodgOvryTykPiv1sFV2WedhwmJoeIPpRMLmHFt4NaoEExIUQNVQx/4V+LZzB6zd2NwMmLgEYrqaTiT8gKM/7TXDQ0xHECVJ3mdt2bnyIH4Z1OtoOpHwE44uPOGDTu+G2YMBBfHLoW5b04mEH3HkAP9Hzl3kw03HKNDadBRR1KkdMGsgqECYtELKTnicIwtv/gZrGPeezWsZTiIuO74FZg6EoDCYnAi1W5lOJPyQ4wovK7eAqasOAPDnYTI6ik84ttG6qDisulV2tZqbTiT8lKMKr8ClufXVrwGIjgiVi459wZFvYc4QCK8FkxIhqonpRMKPOarw4mds4GxGDgBrf3On4TSCQ6th7nCoVg8mr4TIhqYTCT/nmMJbf/Ac3+w/C1hlFxzomFX3Tfu/gg9GQmRjaze2uoxFKLzPMZ/6jzYlAfDm6M7UqyETbBu17zOYPxpqtYRJyyGijulEwiEcU3gnzmdxXfUw4jrHmI7ibHuWQcI4qNMW4pdC1WjTiYSDOOao/Zajqbjkujuzdi6y5o2N6QrjPoYqkaYTCYdxxBbe0XOZ5OS7aCsDBZiz7UNYeB807A7jF0nZCSMcUXhrD1onK0bGyllAI76bC5/8HBrfbG3Zhck/PMIMRxTeJXJnhQGbZsCSR6DZHdbgnaERphMJB3NU4YUFy2CflWr9P2H5E9DyHmtY9pBw04mEwzmi8L7YfcZ0BOf59h+w8iloM9CacCc4zHQiIZxxlvbLPacBqFlVxr+rFKtfh//8EdoOgeHTITDYdCIhAAcU3q4TaQD0blNHdmm9TWv47yuw6hXoMAqGvAeBfv8rJmzE738b3/rqBwBGxjYwnMTPaQ1f/R6+eQM6j4PB/4AA+QdG+Ba3juEppfoqpfYqpfYrpZ65xnI3KKUKlFIjPBexYr7aYx2/69VGbl/yGq3hs2etsus2GQa/LWUnfFKphaeUCgTeAfoBbYExSqkrhqItXO4vwGeeDlleGTn55Ls09WuEERokH0CvcLkg8UlY9w50/zkMfAMCHHEuTNiQO7+Z3YH9WuuDWutcIAGIK2G5x4CFgM+cEt194gIAw7vJ7qxXuFyw/Jew8X246VHo9xdQynQqIa7KncKLAY4VeZxU+L3LlFIxwFBgqueiVdyGQ+cA6NxQbmPyOFeBdUHxlllw66/g7pek7ITPc6fwSvotLn4X/t+Bp7XWBdd8IaWmKKU2KaU2JScnu5ux3DYcTgWge9OaXn8vRynIt24V2zYP7vgt3PmclJ2wBXfO0iYBRW9CbQCcKLZMLJCgrF/6aKC/Uipfa7246EJa62nANIDY2FivD13yzQ9WqUbIZNueU5BnDQKwewn0fgFu/T/TiYRwmztNsBFoqZRqChwHRgNjiy6gtW566Wul1ExgefGyq2x5BS5cGjrE1EDJ1odn5OfAgsmwdwXc/Sfo+ajpREKUSamFp7XOV0o9inX2NRCYobXepZR6sPB5nzpud8nRlEwA2tWXkTk8Ii8bPpoAP3wO/V6DG6eYTiREmbm1r6e1TgQSi32vxKLTWk+qeKyKW38wBYAujeSERYXlZkLCGDi4Cgb+HWInm04kRLn47cGtz3efAuDGpjIkVIXkZFjzTxz+BuLegS7jTCcSotz8svDOXMjmv3utExYNomTCnnLLvmDNLJa0AYa9Dx1Hmk4kRIX4ZeFdmo7xl31aEiTTMZZP1nlrztiTW2HEDGg31HQiISrMLwtv7rojANzT7jrDSWwqMwXmDIHTu2HUbGgzwHQiITzC7wpv3+l0thw9D0CLOjKceJllJFtld/YHGD0PWt1tOpEQHuN3hbdoy3EAnh/YlmDZnS2b9FMwOw5Sj8DYBGh+p+lEQniU3xXepftnh3aRCbfL5MIJmDUILpyEcQug6a2mEwnhcX5XeDn5LtrHVCdKhnN33/ljVtldPAvjF0Ljm0wnEsIr/Gqf71hKJrtOXMDlMp3ERlIOwb/7WycqJi6WshN+za+28J74cCsA/drL2Vm3nDtgbdnlZUL8EqjfxXQiIbzKbwrP5dJsOmINB/VwrxaG09hA8l6YNRhceRC/DK7rYDqREF7nN4X33THrUpR72tUlMEBGR7mm07th9mBAwaQVUOd604mEqBR+cwxv/5l0ACb0aGI2iK87uQ1mDoCAIJicKGUnHMVvCm/JVmtM0kY1ww0n8WHHN1vH7ILDrS276JamEwlRqfxmlzY6IhSARrWk8Ep0dD18MAKqRFnH7KIam04kRKXziy08rTVLt52gsZRdyQ7/D+YOg6rR1m6slJ1wKL8ovAvZ+QAEycmKKx38r7VlV70+TEqEGjJlpXAuvyi8LYWXowzrKh/mn9j/Jcy7F6KaWMfsqtcznUgIo/yi8F5asRuAHs1kOsbL9q6E+WOsExPxyyGijulEQhjnF4VXJSQQgK6Nogwn8RG7l8KH46FuO5i4FKrKMPdCgJ8UnkJxZ5s6Mh0jwM6FsGCSdZvYxCUQLlu9QlziF4UnCm1LgIX3Q8MbYcInEFbDdCIhfIoUnr/YMgc+eRCa3ALjP4bQaqYTCeFzbF94ZzNy2HE8Da216SjmbJwOSx+1Rige+xGEVDWdSAifZOvCK3BpYl/6EoAApx6/W/cerPgVtOprzUERLNNSCnE1ti6846lZgDX37FtjHDiW2//ehE+fgTYDYdQcCA4znUgIn2bre2lzC6yhjZ/o04qqobZelbJb9Rp8/RK0GwbDpkFgsOlEQvg8W7fE96cuAFDgpON3WsPXL8PqV6HjaIh7BwJt/WMUotLY+pNy6bhdxwYOufxCa/jyBWtXtst4GPQWBASaTiWEbdi68Gb+7zAAQQG2PhTpHq3hs9/Cunch9mfQ/6/ghPUWwoNsXXhnM3IAaF7bzy/DcLlg5ZPW5Sc3Pgh9XwGnnpUWogJsXXhn0nOoUy3Uv28pc7lg+eOwZTb0/AXc9QcpOz5PjBwAAAxCSURBVCHKydaFFxYcwC0tok3H8B5XASx5BLbNh9uehF7PStkJUQG2LrwApQgN9tPjWAX58MkUazCAXs/C7U+ZTiSE7dm68PxWfi4svA/2LIU+L8ItT5hOJIRfsG3h5Re4OJOeg99dgpefAx/Fw76VcM/LcNMjphMJ4TdsW3ipmXmmI3heXpY1cOf+L6H/69D9AdOJhPArti28S9rF+MlFx7kXrSHZD622LijuFm86kRB+x7aF9/6agwD4xURlOenWZDtH18KQd6HzWNOJhPBLtiw8l0szbbVVeHe3vc5wmgrKToMPRkLSJhj2PnQYYTqREH7LrWs6lFJ9lVJ7lVL7lVLPlPD8OKXU9sL/vlVKdfJ81B9l5RUA0KlhJLWrhXrzrbwrKxVmD4Hjm2Hkv6XshPCyUrfwlFKBwDvAXUASsFEptVRrvbvIYoeA27XWqUqpfsA04EZvBC5qQAcbb91dPAdzhkDy99ZYdm36m04khN9zZwuvO7Bfa31Qa50LJABxRRfQWn+rtU4tfLgO8OqM2PtOpwOQ77LpNSkZyTBrECTvtUYplrITolK4U3gxwLEij5MKv3c19wErKxKqNOsPpQDQLDrCm2/jHemnYOYASDkIYz+ElneZTiSEY7hz0qKk86AlbloppXphFd4tV3l+CjAFoFGjRm5GvNKO42kA9GhmszlX045bW3bpp6yZxZqU+NckhPASd7bwkoCGRR43AE4UX0gp1RGYDsRprc+V9EJa62la61itdWzt2rXLkxeAtQesl68WZqNhzc8fhZn9IeMMTFgkZSeEAe5s4W0EWiqlmgLHgdHATy4UU0o1AhYBE7TW+zyespi61cOoWTWEQLtchJdyEGYNhuwLMHEJNOhmOpEQjlRq4Wmt85VSjwKfAYHADK31LqXUg4XPTwWeB2oB7xaOTZevtY71Vug9Jy/Q5/o63np5zzq739qNzc+C+KVQv7PpREI4llsXHmutE4HEYt+bWuTr+4H7PRvt2grscIb2zPcwe7A1rl38criuvelEQjiabQeT69gg0nSEazu10zobCzBphZSdED7AdoWXkZNvOkLpTmyFWQMhMAQmJUKdNqYTCSGwYeF9vusUAFVDfXR6wqTN1m5sSARMXgHRLUwnEkIUsl3hfXf0PAADOtY3nKQER9fB7DgIi4TJiVCzmelEQogibFd4ly46ruNrgwYc/gbmDIOIOjB5JUSW/8JqIYR32K7wLl16FxzoQ9EPfA1zR0CNBtaWXY1r3XknhDDFh1rDPftOZ3Bzi1qmY/zohy+swTtrNrPOxlaz8QguQvg5WxXeifNZZOTkc95X5rP4PhESxkLtVhC/DCLKf7ucEML7bFV42YUDf47v0dhwEmD3EvhoAtRtb5VdVR/a6hRClMhWhXdpHLwQ08fvdnwMCyZDTDeYuBiqRJnNI4Rwi60KL6/Aup2seR2D4+BtnQeLHoBGPWD8Qgjzk1nThHAAWxXeJRGmLjrePAsWPwxNboVxCyC0mpkcQohysWXhGbHhfVj2C2jR2xqpOKSq6URCiDKSwnPH2nch8dfQqp81B0VwFdOJhBDlYKvCW7nzZOFXlTjw5zdvwGe/gesHw6jZEORjd3gIIdxmm4m48wtcJO6wBg5oGl1Ju5OrXoWv/wTth8PQaRBom78uIUQJbPMJPpuRC0CXRpHeH9pda6voVr8GHUfDkHchwEdHZxFCuM02hXfJqNiGpS9UEVrDF8/Dt29Blwkw6E0pOyH8hO0Kz6u0hk9/A+vfg9j7oP/rEGCrw5xCiGuQwrvE5YLEX8GmGdDjYbjnZVA2mRVNCOEW2xReamau917cVWBdY/fdXLj5cejzeyk7IfyQbQpvyVZr7u/wEA8fTyvIhyUPw/YP4banoNdvpeyE8FO2KbzvjqYC0L9DPc+9aEEeLJoCuxZBr9/B7U967rWFED7HNoUXFR4CeHCk4/xc+HgyfL8c7vqDtSsrhPBrtik8gNZ1PXSzfl42LIiHfZ9C31egx0OeeV0hhE+zzTUXO46nodEVf6G8LGuU4n2fwoC/StkJ4SC22cIrcGlOX8ip2IvkXoT5o+HQGhj8D+g60TPhhBC2YJvCCw5S9Gxep/wvkJMOH4yCY+tg6FToNNpz4YQQtmCbwquQ7DRrGsXjm2H4dGswACGE4/h/4WWmwNxhcGoHjJwJbQebTiSEMMS/C+/iOZgTB8l74d650Lqf6URCCIP8t/AyzsDsOEg5CKPnQ8s+phMJIQzzz8K7cBJmD4bzx6z5J5rdYTqREMIH+F/hpSXBrEGQftqaRrHJzaYTCSF8hH8VXuoRq+yyUmHCJ9DoRtOJhBA+xH8KL+UgzBwEuekwcTHEdDOdSAjhY/yj8M7+YG3Z5edA/DKo18l0IiGED7J/4Z3ZA7MGAxomLYe67UwnEkL4KNsMHlCiUztg5gBrwM5JK6TshBDXZN/CO/EdzBwIgaEwKRFqtzadSAjh4+xZeEmbYFYchFaHyYkQ3cJ0IiGEDbhVeEqpvkqpvUqp/UqpZ0p4Ximl3ip8frtSqqvnoxY6shZmD4HwKJi8Amo29dpbCSH8S6mFp5QKBN4B+gFtgTFKqbbFFusHtCz8bwrwnodzAtD84ncwdzhUqwuTV0JkI2+8jRDCT7mzhdcd2K+1Pqi1zgUSgLhiy8QBs7VlHRCplPLgbDsQW7CVB449DZENrWN21et78uWFEA7gTuHFAMeKPE4q/F5Zl0EpNUUptUkptSk5Odn9lHlZPJv7FmdDYiB+ubWFJ4QQZeTOdXglTdJafHIJd5ZBaz0NmAYQGxvr/gQVwVXIGJ5AcFQDiKjt9h8TQoii3Cm8JKBhkccNgBPlWKZCmrTv4cmXE0I4kDu7tBuBlkqppkqpEGA0sLTYMkuBiYVna3sAaVrrkx7OKoQQFVLqFp7WOl8p9SjwGRAIzNBa71JKPVj4/FQgEegP7AcygcneiyyEEOXj1r20WutErFIr+r2pRb7WwCOejSaEEJ5lzzsthBCiHKTwhBCOIYUnhHAMKTwhhGNI4QkhHEMKTwjhGFJ4QgjHUNYldAbeWKlk4EgZ/1g0cNYLcUzwl3Xxl/UAWRdfVdZ1aay1LvGme2OFVx5KqU1a61jTOTzBX9bFX9YDZF18lSfXRXZphRCOIYUnhHAMuxXeNNMBPMhf1sVf1gNkXXyVx9bFVsfwhBCiIuy2hSeEEOXmk4XnU9NCVoAb6zGuMP92pdS3SqlOJnK6o7R1KbLcDUqpAqXUiMrMVxburItS6g6l1Fal1C6l1KrKzugON36/aiillimlthWuh8+OU6mUmqGUOqOU2nmV5z3zmdda+9R/WIOMHgCaASHANqBtsWX6Ayux5tLoAaw3nbuc69ETiCr8up8vroe761Jkuf9gjZ04wnTuCvxcIoHdQKPCx3VM5y7nevwW+Evh17WBFCDEdParrM9tQFdg51We98hn3he38HxiWkgPKHU9tNbfaq1TCx+uw5oLxBe58zMBeAxYCJypzHBl5M66jAUWaa2PAmitfXF93FkPDVRTSikgAqvw8is3pnu01qux8l2NRz7zvlh4HpsW0rCyZrwP618wX1TquiilYoChwFR8mzs/l1ZAlFLqv0qpzUqpiZWWzn3urMfbwPVYE2rtAB7XWrsqJ57HeeQz79YQ75XMY9NCGuZ2RqVUL6zCu8WricrPnXX5O/C01rrA2qDwWe6sSxDQDegNVAHWKqXWaa33eTtcGbizHvcAW4E7gebAF0qpNVrrC94O5wUe+cz7YuH5xLSQHuBWRqVUR2A60E9rfa6SspWVO+sSCyQUll000F8pla+1Xlw5Ed3m7u/XWa31ReCiUmo10AnwpcJzZz0mA69o6yDYfqXUIaANsKFyInqUZz7zpg9WlnBwMgg4CDTlx4Ox7YotM4CfHsDcYDp3OdejEdZMbz1N563ouhRbfia+e9LCnZ/L9cBXhcuGAzuB9qazl2M93gNeLPy6LnAciDad/Rrr1ISrn7TwyGfe57bwtJ9MC+nmejwP1ALeLdwyytc+eMO3m+tiC+6si9Z6j1LqU2A74AKma61LvFzCFDd/Jn8EZiqldmAVxdNaa58cQUUpNR+4A4hWSiUBLwDB4NnPvNxpIYRwDF88SyuEEF4hhSeEcAwpPCGEY0jhCSEcQwpPCOEYUnhCCMeQwhNCOIYUnhDCMf4f1jE3vRNHVzcAAAAASUVORK5CYII=\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_curve # 使用sklearn库计算roc曲线\n",
    "fpr,tpr,thresholds = roc_curve(y_test,y_pred)\n",
    "plt.figure(figsize=(5,5))\n",
    "plt.plot(fpr,tpr)\n",
    "plt.plot([0,1],[0,1])\n",
    "print(thresholds)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T02:12:11.568665300Z",
     "start_time": "2023-12-09T02:12:11.365661300Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## AUC"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "outputs": [
    {
     "data": {
      "text/plain": "0.8563545504741993"
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import auc\n",
    "auc(df_scores.fpr,df_scores.tpr)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T02:16:46.288786Z",
     "start_time": "2023-12-09T02:16:46.246843600Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "outputs": [
    {
     "data": {
      "text/plain": "0.856455259255658"
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 只需要AUC\n",
    "from sklearn.metrics import roc_auc_score\n",
    "roc_auc_score(y_test,y_pred)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T02:18:00.817119600Z",
     "start_time": "2023-12-09T02:18:00.773158900Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    ">- ROC曲线和AUC分数表示模型分离正类和父类的程度,AUC有一个很好的概率解释,它告诉我们随机选择正类得分高于随机选择负类的概率是多少?.即随机选择的流失客户的分数高于随机选择的未流失客户分数的概率"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 验证AUC的解释性"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "outputs": [
    {
     "data": {
      "text/plain": "0.8579"
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neg = y_pred[y_test==0] # 选择所有没有流失客户的分数\n",
    "pos = y_pred[y_test==1] # 选择所有流失客户的分数\n",
    "\n",
    "np.random.seed(1)\n",
    "neg_choice = np.random.randint(low=0,high=len(neg),size=10000) # 随机选择10000个负类\n",
    "pos_choice = np.random.randint(low=0,high=len(pos),size=10000) # 随机选择10000个正类\n",
    "(pos[pos_choice]>neg[neg_choice]).mean()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T02:30:09.792525200Z",
     "start_time": "2023-12-09T02:30:09.739513100Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 4.4.参数调优"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "outputs": [],
   "source": [
    "def train(df,y):\n",
    "    cat = df[categorical+numerical].to_dict(orient='records')\n",
    "    dv = DictVectorizer(sparse=False) # 独热编码转化器\n",
    "    dv.fit(cat)\n",
    "    X = dv.transform(df)\n",
    "    model = LogisticRegression(solver='liblinear') # 模型\n",
    "    model.fit(X,y)\n",
    "    return dv,model"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T02:42:52.110664300Z",
     "start_time": "2023-12-09T02:42:52.065657700Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "outputs": [],
   "source": [
    "def predict(df,dv,model):\n",
    "    cat = df[categorical+numerical].to_dict(orient='records') # 应用与训练中相同的独热编码\n",
    "    X = dv.transform(cat)\n",
    "    y_pred = model.predict_proba(X)[:,1] # 使用该模型进行预测\n",
    "    return y_pred"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-12-09T02:52:38.105609700Z",
     "start_time": "2023-12-09T02:52:38.058568500Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
